use hahn;
pragma yt.Pool="robot-webmaster";

$misleading = (
    select CandidateId
    from (
        select 
            AGGREGATE_LIST(CandidateId) as CandidateId
        from `//home/webmaster/users/lester/SS15208/dataset_full`
        group by TextTokens
        having count(distinct Resolution) > 1
    ) flatten list by CandidateId
);

insert into `//home/webmaster/users/lester/SS15208/dataset_full_dl`
    with truncate
select Resolution, TextTokens
from `//home/webmaster/users/lester/SS15208/dataset_full`
where length(TextTokens) > 20 and CandidateId not in $misleading;
