User:AnomieBOT/source/tasks/SourceUploader.pm

package tasks::SourceUploader;

use strict;

use AnomieBOT::Task;
use vars qw/@ISA/;
@ISA=qw/AnomieBOT::Task/;

use POSIX qw/strftime/;
use Data::Dumper;
use Fcntl ':mode';

my %extensions=(
    'pl' => 'perl',
    'pm' => 'perl'
);

my @sizes=('', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB');

sub approved {
    # Per [[WP:BOT#Approval]], any bot or automated editing process that only
    # affects only the operators' user and talk pages (or subpages thereof),
    # and which are not otherwise disruptive, may be run without prior
    # approval.
    return 1;
}

sub new {
    my $class=shift;
    my $self=$class->SUPER::new();
    $self->{'pages'}={};
    $self->{'loadexisting'}=1;

    my $basedir=$0;
    $basedir=~s{/[^/]*$}{};
    if(!-d $basedir){
        $self->warn("Cannot find source directory\n");
        return $self;
    }

    my %pages=();
    my @dirs=($basedir);
    while(my $dir=shift @dirs){
        if(!opendir(D, $dir)){
            $self->warn("Cannot open directory $dir: $!\n");
            return $self;
        }
        my $dirpage=substr($dir,length($basedir));
        $dirpage='/'.$dirpage if(substr($dirpage,0,1) ne '/');

        $dirpage ="==Index of $dirpage==\n";
        $dirpage.="{| class=\"wikitable sortable\" style=\"width:100%\"\n";
        $dirpage.="! Filename !! Size !! Modified\n";
        while(my $page=readdir(D)){
            next if substr($page,0,1) eq '.';

            my $p="$dir/$page";
            my $pp=substr($p,length($basedir));
            my @stat=stat($p);
            my $img='Gnome-fs-executable.svg';
            if(-d $p){
                next if ($stat[2]&(S_IROTH|S_IXOTH))!=(S_IROTH|S_IXOTH);
                push @dirs, $p;
                $img='Gnome-fs-directory-visiting.svg';
            } elsif(-f $p){
                next if ($stat[2]&(S_IROTH))!=(S_IROTH);
                if(!open(X, '<:utf8', $p)){
                    $self->warn("Cannot open file $p: $!\n");
                    return $self;
                }
                do {
                    local $/=undef;
                    $pages{$pp}=<X>;
                };
                close(X);
                if($page=~/\.([^.]+)$/ && exists($extensions{$1})){
                    $pages{$pp}="<source lang=\"".$extensions{$1}."\">\n".$pages{$pp}."\n</so"."urce>";
                } else {
                    $pages{$pp}="<pre>\n".$pages{$pp}."\n</pre>";
                }
                $img='Gnome-fs-regular.svg';
            } else {
                $self->warn("Unusual filetype on $p\n");
            }
            my $sz=$stat[7];
            my $i=0;
            while($sz>=1024 && $i<@sizes){ $sz/=1024; $i++; }
            ($sz="$sz")=~s/(\.\d\d)\d+/$1/;
            $sz.='&nbsp;'.$sizes[$i] if $i>0;
            $dirpage.="|-\n";
            $dirpage.="| \x5b\x5bImage:$img|32x32px\x5d\x5d";
            $dirpage.=" \x5b\x5b/$page|$page\x5d\x5d\n";
            $dirpage.="|align=\"right\"| \x7b\x7bsort|".sprintf("%020d",$stat[7])."|$sz\x7d\x7d\n";
            $dirpage.="|align=\"center\"| \x7b\x7bsort|".sprintf("%020d",$stat[9]).strftime("|%F %T (UTC)", gmtime($stat[9]))."\x7d\x7d\n";
        }
        closedir(D);
        $dirpage.="|}";
        $pages{substr($dir,length($basedir))}=$dirpage;
    }

    $self->{'pages'}={%pages};
    return $self;
}

sub run {
    my ($self, $api)=@_;

    my @keys=keys(%{$self->{'pages'}});
    return undef unless @keys;

    $api->task('SourceUploader 1.0');
    $api->read_throttle(6);
    $api->edit_throttle(10);
    #return 300 if $self->check_shutoff($api, 'SourceUploader');

    my $src='User:'.$api->user.'/source';

    if($self->{'loadexisting'}){
        my %q=(
            list        => 'allpages',
            apprefix    => $api->user.'/source',
            apnamespace => '2',
            aplimit     => 'max'
        );

        my ($k,$v,$k2,$v2,@x);
        my $res;
        do {
            $res=$api->query(%q);
            if($res->{'code'} ne 'success'){
                $self->warn("Failed to retrieve source tree: ".$res->{'error'});
                return 300;
            }
            if(exists($res->{'query-continue'})){
                while(($k,$v)=each(%{$res->{'query-continue'}})){
                    while(($k2,$v2)=each(%$v)){
                        $q{$k2}=$v2;
                    }
                }
            }
            while(($k,$v)=each(%{$res->{'query'}})){
                if(ref($v) eq 'HASH'){
                    @x=values(%$v);
                } elsif(ref($v) eq 'ARRAY'){
                    @x=@$v;
                } else {
                    next;
                }
                for(my $i=0; $i<@x; $i++){
                    $v2=substr($x[$i]{'title'}, length($src));
                    if(!exists($self->{'pages'}{$v2})){
                        $self->{'pages'}{$v2}='';
                        push @keys, $v2;
                    }
                }
            }
        } while(exists($res->{'query-continue'}));

        $self->{'loadexisting'}=0;
    }

    my $end=time()+300;
    while(@keys){
        last if time()>$end;
        my $k=shift @keys;
        my $tok=$api->edittoken($src.$k);
        next if($tok->{'code'} ne 'success');
        if(exists($tok->{'missing'}) ||
           $tok->{'revisions'}[0]{'*'} ne $self->{'pages'}{$k}){
            my $r=$api->edit($tok, $self->{'pages'}{$k},
                             'Updating published source', 0, 1);
            if($r->{'code'} ne 'success'){
                $self->warn("Write error for $k: ".$r->{'error'});
                next;
            }
        }
        delete($self->{'pages'}{$k});
    }
    return 0;
}