package NWebmaster.proto.urltree;

enum Branch {
    BRANCH_TRUNK    = 1;
    BRANCH_RUS      = 2;
    BRANCH_USA      = 3;
    BRANCH_TUR      = 4;
    BRANCH_DEU      = 5;
}

enum RecordSource {
    SOURCE_WEBMASTER_HOST           = 10;
    SOURCE_USER_PATH                = 20;
    SOURCE_KIWI                     = 30;
    SOURCE_KIWI_TRUNK               = 31;
    SOURCE_KIWI_RUS                 = 32;
    SOURCE_KIWI_USA                 = 33;
    SOURCE_KIWI_TUR                 = 34;
    SOURCE_KIWI_DEU                 = 35;
    SOURCE_GEMINI                   = 40;
    SOURCE_JUPITER_BASE             = 72;
    SOURCE_JUPITER_DIFF             = 73;
    SOURCE_JUPITER_BASE_PRODUCTION  = 74;
    SOURCE_JUPITER_BASE_ACCEPTANCE  = 75;
    SOURCE_JUPITER_DIFF_PRODUCTION  = 76;
    SOURCE_JUPITER_DIFF_ACCEPTANCE  = 77;
    SOURCE_MERGED                   = 99;
}

enum SearchDiffStatus {
    SEARCH_DIFF_STATUS_UNKNOWN  = 0;
    SEARCH_DIFF_STATUS_NEW      = 1;
    SEARCH_DIFF_STATUS_GONE     = 2;
}

enum UrlWMStatus {
    WM_STATUS_UNKNOWN   = 0;
    WM_STATUS_KNOWN     = 1;
    WM_STATUS_NEW       = 2;
}

enum UrlGeminiStatus {
    JUPITER_STATUS_UNKNOWN   = 0;
    JUPITER_STATUS_KNOWN     = 1;
}

message KiwiUrlInfo {
    required Branch branch                          = 1;
    optional uint32 http_code                       = 2;
    optional int32 last_access                      = 3;
    optional int32 add_time                         = 4; //walrus
    optional int32 http_mod_time                    = 5;
    optional uint32 size                            = 6;
    optional uint32 flags                           = 7;
    optional uint32 mime_type                       = 8;
    optional int32 encoding                         = 9;
    optional uint32 language                        = 10;
    optional uint32 status                          = 11; //walrus
    optional int32 mod_time                         = 12;
    optional UrlFaviconInfo favicon                 = 13;
    optional int64 fetch_time                       = 14;
    optional AntispamInfo antispam                  = 15;
    optional PageContentInfo page_content_info      = 16;
    optional bool is_searchable                     = 17; //walrus
    optional string redir_target                    = 18;
    optional uint32 robot_base_last_access          = 19; //walrus
    optional uint32 search_base_last_access         = 20; //walrus
    optional string search_base_main_url            = 21; //walrus
    optional uint32 source_id                       = 22;
    optional uint32 jupiter_url_status_excluded     = 23;
    optional uint32 jupiter_url_status_searchable   = 24;
    optional uint64 oldwm_url_id                    = 40;
}

message RawExternalsInfo {
    optional uint32 http_code                   = 2;
    optional int32 last_access                  = 3;
    optional uint32 mime_type                   = 8;
    optional int32 encoding                     = 9;
    optional UrlFaviconInfo favicon             = 13;
    optional int64 fetch_time                   = 14;
    optional AntispamInfo antispam              = 15;
    optional PageContentInfo page_content_info  = 16;
    optional bool is_searchable                 = 17;
    optional string search_base_main_url        = 18;
    optional bool from_jupiter                  = 19;
    required uint32 timestamp                   = 100;
}

message HostStatInfo {
    optional int64 num_of_docs                      = 1;
    optional int64 num_of_docs_on_search            = 2;
    optional int64 num_of_docs_empty_meta_descr     = 3;
    optional int64 num_of_docs_empty_title          = 4;
    optional int64 num_of_docs_processed            = 5;
    optional int64 num_of_docs_html                 = 6;
    optional int64 num_of_docs_encoding_unknown     = 7;
    optional int64 num_of_docs_encoding_unsupported = 8;
    optional int64 average_fetch_time               = 20; //ms
    repeated string samples_empty_meta_descr        = 53;
    repeated string samples_empty_title             = 54;
    repeated string samples_slow_fetch              = 55; //fetch time, greater than 3 seconds
    repeated SlowUrlSample slow_url_samples         = 56;
}

message SlowUrlSample {
    optional string path                            = 1;
    optional int64  last_access                     = 2;
    optional int64  fetch_time                      = 3;
}

message KiwiHostInfo { // related table prod/kiwi/hosts-merged
    optional int32 add_time                 = 1;
    optional int32 last_access              = 2;
    optional uint32 host_status             = 3;
    optional uint32 host_penalty            = 4;
    optional uint32 robots_txt_http_code    = 5;
    optional string robots_txt_content      = 6;
}

message KiwiUrlRecordInfo {
    required KiwiUrlInfo url_info               = 1;
    optional uint32 spider_timestamp            = 2 [ default = 0 ];
    optional uint32 walrus_timestamp            = 4 [ default = 0 ];
    optional UrlWMStatus url_wm_status          = 10 [ default = WM_STATUS_UNKNOWN ];
    optional UrlGeminiStatus url_jupiter_status  = 11 [ default = JUPITER_STATUS_UNKNOWN ];
}

message KiwiHostRecordInfo {
    required KiwiHostInfo host_info = 1;
    required uint32 timestamp       = 2;
}

message SearchBaseInfo {
    optional bool from_jupiter                      = 51;
    optional SearchDiffStatus search_diff_status    = 52;
}

message RecordSourceInfo {
    optional string path                            = 1;
    repeated string user_path                       = 20;
    repeated KiwiUrlInfo url_info                   = 40;
    optional KiwiUrlInfo latest_url_info            = 41;
    optional SearchBaseInfo search_base_production  = 53;
    optional SearchBaseInfo search_base_acceptance  = 54;
    optional string gemini_double                   = 60;
    optional uint64 turbo_source_flags              = 71;
}

message UrlFaviconInfo {
    optional string url         = 1;
    optional bool is_default    = 2;
}

enum RobotsTxtErrorId {
    ERROR_RULE_NOT_SLASH            = 1;
    ERROR_ASTERISK_MULTI            = 2;
    ERROR_HOST_MULTI                = 3;
    ERROR_ROBOTS_HUGE               = 4;
    ERROR_RULE_BEFORE_USER_AGENT    = 5;
    ERROR_RULE_HUGE                 = 6;
    ERROR_HOST_FORMAT               = 7;
    ERROR_TRASH                     = 8;
    ERROR_SITEMAP_FORMAT            = 9;
    ERROR_CRAWL_DELAY_FORMAT        = 10;
    ERROR_CRAWL_DELAY_MULTI         = 11;
    ERROR_CLEAN_PARAM_FORMAT        = 12;

    WARNING_EMPTY_RULE              = 30;
    WARNING_SUSPECT_SYMBOL          = 31;
    WARNING_UNKNOWN_FIELD           = 33;
    WARNING_UPPER_REGISTER          = 34;
    WARNING_SITEMAP                 = 35;

    UNKNOWN_ERROR                   = 100;
}

message RobotsTxtError {
    required RobotsTxtErrorId id    = 1;
    required uint32 line_no         = 2;
}

message RobotsTxtParsedInfo {
    optional string host            = 1;
    repeated string clean_params    = 2;
    repeated string sitemaps        = 3;
    optional bool is_disallow_all   = 4;
    repeated int32 accepted_lines   = 5;
    optional string host_2nd_level              = 6; //host directive of host specified in origin host directive WMC-4192
    optional bool host_2nd_level_found_in_db    = 7;
}

message RobotsTxtInfo {
    optional uint32 http_code           = 1;
    optional string content             = 2;
    repeated RobotsTxtError errors      = 10;
    repeated RobotsTxtError warnings    = 20;
    optional RobotsTxtParsedInfo parsed = 30;
}

message WhoisMessage {
    enum FoundState {
        FOUND       = 0;
        NOT_FOUND   = 1;
        REDIRECT    = 2;
    }; 

    optional FoundState found       = 1 [ default = NOT_FOUND ];
    optional fixed32 time_created   = 4;
    optional fixed32 time_expired   = 5;
    optional fixed32 time_updated   = 6;
    optional string registrar       = 7;
    optional string whois_server    = 8;

    message UserInfo {
        optional string name        = 1;
        optional string email       = 2;
        repeated string phones      = 4;
    }

    repeated UserInfo users         = 11;
    optional fixed32 time_verisign  = 12;
    optional fixed32 time_parsed    = 13;
    optional string host            = 14;
}

message FaviconInfo {
    optional string face_url    = 1;
    repeated string known_urls  = 2;
}

message AntispamInfo {
    optional bool pharma    = 1;
    optional bool bad_forum = 2;
}

message PageContentInfo {
    optional string meta_description_json   = 1;
    optional string title                   = 2;
    optional string tag_meta_descr          = 50;
}
