Wikipedia

User:AnomieBOT/source/tasks/SourceUploader.pm: Difference between revisions

Browse history interactively
← Previous edit
Content deleted Content added
VisualWikitext
AnomieBOT (talk | contribs)
Bots
6,859,315 edits
Updating published sources: General: * Have bot.pl set $0, for more useful ps output. * New utility function to return the bot base directory. SourceUploader: * Use said new utility function.
AnomieBOT (talk | contribs)
Bots
6,859,315 edits
Updating published sources: SourceUploader: * Improve list handling for doc pages. * Add a TemplateStyles stylesheet to doc pages. * Cleanup some manpage refs.
 
(44 intermediate revisions by 8 users not shown)
Line 1:
{{ombox|type=notice|text= Per [[WP:BOT#Approval]], any bot or automated editing process that only affects only the operators' user and talk pages (or subpages thereof), and which are not otherwise disruptive, may be run without prior approval.}}
<sourcesyntaxhighlight lang="perl">
package tasks::SourceUploader;
 
Line 7:
=begin metadata
 
Bot: AnomieBOT
Task: SourceUploader
BRFA: N/A
Status: Begun 2008-08-15
Rate: On bot startup only, max 6 edits/minute
Created: 2008-08-16
 
Line 23:
use strict;
 
use AnomieBOT::Task qw/:time/;
use vars qw/@ISA/;
@ISA=qw/AnomieBOT::Task/;
 
use POSIXStorable qw/strftimedclone/;
use PodURI::Simple::WikiEscape;
use tasks::SourceUploader::WikiPod;
use tasks::SourceUploader::Pod;
use Data::Dumper;
Line 38 ⟶ 39:
'ini' => 'ini',
'sh' => 'bash',
'css' => 'css',
'js' => 'js',
'json' => 'json',
'html' => 'html',
);
 
Line 60 ⟶ 65:
my $class=shift;
my $self=$class->SUPER::new();
$self->{'pages'}={};
$self->{'loadexisting'}=1;
$self->{'order'}=-1000;
$self->{'fail'}=0;
return $self;
}
 
sub build_sources {
my $self = shift;
 
my $basedir=$AnomieBOT::TaskAPI::basedir;
$basedir=~s{/$}{};
if(!-d $basedir){
$self->{'fail'}=1;
AnomieBOT::API->warn("Cannot find source directory\n");
return $self;
}
 
$self->{'summary'}='Updating published sources: ';
if(!open(X, "<:utf8", 'ChangeLog')){
$self->warn("Cannot load changelog\n"){'fail'}=1;
AnomieBOT::API->warn("Cannot load changelog: $!\n");
return $self;
return;
}
local $_;
Line 91 ⟶ 103:
$self->{'summary'}=$self->{'summary'};
 
my $mainbot="\x02BOT\x03";
my %pages=();
my %tasks=(''=>{
'01 Current' => [],
'0405 Awaiting approval' => [],
'0506 Past' => [],
});
$tasks{'TaskList'}=dclone($tasks{''});
my @dirs=($basedir);
$self->{'shutoff_pages'}=[];
$self->{'check_shutoff_notice'}=[];
while(my $dir=shift @dirs){
if(!opendir(D, $dir)){
$self->{'fail'}=1;
AnomieBOT::API->warn("Cannot open directory $dir: $!\n");
return $self;
}
my $dirpage=substr($dir,length($basedir));
$dirpage='/'.$dirpage if(substr($dirpage,0,1) ne '/');
 
$dirpagemy ="==Index of $@dirpage==\n"();
$dirpage.="{| class=\"wikitable sortable\" style=\"width:100%\"\n";
$dirpage.="! Filename !! Size !! Modified\n";
while(my $page=readdir(D)){
next if substr($page,0,1) eq '.';
Line 123 ⟶ 136:
next if ($stat[2]&(S_IROTH))!=(S_IROTH);
if(!open(X, '<:utf8', $p)){
$self->warn("Cannot open file $p: $!\n"){'fail'}=1;
returnAnomieBOT::API->warn("Cannot open file $p: $self!\n");
return;
}
do {
Line 133 ⟶ 147:
my $top='';
if($page eq 'ChangeLog'){
# Pull out most recent 64K10M of changelog entries
my @x=split(/\n/, $pages{$pp});
$pages{$pp}='';
while(defined(my $x=shift @x)){
last if(length($pages{$pp})>65536204800 && $x=~/^==.*==$/);
$pages{$pp}.=$x."\n";
}
$pages{$pp}.="\n__NOTOC__\n";
$pages{$pp}.="\x7b\x7bombox|type=notice|text=See the \x7b\x7bsubst:history|\x7b\x7bsubst:FULLPAGENAME\x7d\x7d|page history|subst=subst:\x7d\x7d for further ChangeLog entries\x7d\x7d\n" if @x;
} elsif($page=~/\.([^.]+)$/ && exists($extensions{$1})){
if($extensions{$1} eq 'perl'){
# Try to construct POD documentation
my $parser =Pod tasks::SimpleSourceUploader::WikiWikiPod->new('mediawiki'"User:$mainbot/source");
my $x='';
$parser->output_string(\$x);
$parser->parse_string_document($pages{$pp});
$pages{"$pp/doc"}="<templatestyles src=\"User:$mainbot/source-docs.css\"/>\n$x" if($parser->content_seen);
 
# Handle embedded notices and metadata
Line 160 ⟶ 175:
$x ="<noinclude>\n";
$x.="{| class=\"wikitable\"\n";
$x.="! Account !! Task !! Disable !! Status{{tlx|bots}} !! RateStatus !! Description\n";
$x.="</noinclude>\n";
$x.="|- valign=\"top\"\n";
my $task=$metadata{'task'};
my $bot=$metadata{'bot'} // "\x02BOTx02!!!\x03";
$x.="{{#if:{{{noacct|}}}||{{!}}align=\"center\"|{{!}} [[User:$bot/source$pp|$taskbot]]}}\n";
$x.="|align=\"center\"| <span class=\"plainlinks\">[{{fullurl:[User:$bot/shutoffmainbot/source$taskpp|action=edit}} Here$task]]</span>\n";
if(exists($metadata{'shutoff'}) && $metadata{'shutoff'} eq 'false'){
$x.="|align=\"center\"| No\n";
} else {
$x.="|align=\"center\"| <span class=\"plainlinks\">[{{fullurl:User:$bot/shutoff/$task|action=edit}} Here]</span>\n";
push @{$self->{'shutoff_pages'}}, "User:$bot/shutoff/$task";
push @{$self->{'check_shutoff_notice'}}, $bot;
}
if(exists($metadata{'exclusion'}) && $metadata{'exclusion'} eq 'false'){
$x.="|align=\"center\"| {{N}}\n";
} else {
$x.="|align=\"center\"| {{Y}}\n";
}
my $brfa=$metadata{'brfa'};
my $status=$metadata{'status'};
Line 179 ⟶ 206:
$x.="<p style=\"margin:0;padding:0;font-size:smaller\">Supplemental:<br />\n";
for(my $i=0; $i<@{$metadata{'+brfa'}}; $i++){
my $x.s="+ \x5b\x5b".$metadata{'+brfa'}[$i].'|'.$metadata{'+status'}[$i]."\x5d\x5d<br />\n";
$s=$1 if $s=~/^Approved (\d{4}-\d{2}-\d{2})$/;
$x.="+ \x5b\x5b".$metadata{'+brfa'}[$i].'|'.$s."\x5d\x5d<br />\n";
}
$x.="</p>\n";
}
$x.="| ".$metadata{'rate'}."\n";
$x.="|\n".$metadata{'*'}."\n";
$x.="<noinclude>\n";
Line 190 ⟶ 218:
$pages{"$pp/metadata"}=$x;
my $section=determine_task_section(".$pp", %metadata);
$tasks{"TaskList"}{$section}//=[] unless exists($tasks{$section});
push @{$tasks{"TaskList"}{$section}}, $metadata{'created'}." {{User:$botmainbot/source$pp/metadata}}";
$tasks{"TaskList/$bot"}//=dclone($tasks{''});
$tasks{"TaskList/$bot"}{$section}//=[];
push @{$tasks{"TaskList/$bot"}{$section}}, $metadata{'created'}." {{User:$mainbot/source$pp/metadata|noacct=1}}";
}
}
$pages{$pp}="<sourcesyntaxhighlight lang=\"".$extensions{$1}."\">\n".$pages{$pp}."\n</sosyntax"."urcehighlight>";
} else {
$pages{$pp}="<pre>\n".$pages{$pp}."\n</pre>";
Line 202 ⟶ 233:
$img='Gnome-fs-regular.svg';
} else {
$selfAnomieBOT::API->warn("Unusual filetype on $p\n");
}
my $sz=$stat[7];
Line 209 ⟶ 240:
($sz="$sz")=~s/(\.\d\d)\d+/$1/;
$sz.='&nbsp;'.$sizes[$i] if $i>0;
my $dirpage.="|-\n";
$dirpage.="| \x5b\x5bImage:$img|32x32px\x5d\x5d";
$dirpage.=" \x5b\x5b/$page|$page\x5d\x5d\n";
$dirpage.="|align=\"right\"| \x7b\x7bsort|".sprintf("%020d",$stat[7])."|$sz\x7d\x7d\n";
$dirpage.="|align=\"center\"| \x7b\x7bsort|".sprintf("%020d",$stat[9]).strftime("|%F %T (UTC)", gmtime($stat[9]))."\x7d\x7d\n";
push @dirpage, { ts=>$stat[9], text=>$dirpage };
}
closedir(D);
my $dirpage=substr($dir,length($basedir));
$dirpage='/'.$dirpage if(substr($dirpage,0,1) ne '/');
$dirpage ="==Index of $dirpage==\n";
$dirpage.="{| class=\"wikitable sortable\" style=\"width:100%\"\n";
$dirpage.="! Filename !! Size !! Modified\n";
$dirpage.=join "", map $_->{'text'}, sort { $b->{'ts'}<=>$a->{'ts'} } @dirpage;
$dirpage.="|}";
$pages{substr($dir,length($basedir))}=$dirpage;
}
 
my $tasklist%tasklists=''();
foreach my $sectionpg (sort keys %tasks) {
mynext @links=sortif @{$tasks{$section}}pg eq '';
$section=~s/^my (\d+$tl,$acct) //;
my if($npg=~/^TaskList\/(.*)$1;/){
$tl="<noinclude>\nThis page lists basic information about every task coded for [[User:$1|$1]], including links to the bot approval and individual shutoff pages. For the full list of tasks for all associated bot accounts, see [[User:$mainbot/TaskList]].\n\n== Tasks ==\n</noinclude>";
$tasklist.="\n" if $tasklist ne '';
$tasklist.="=== $section ==acct=\n"'';
if(@links){
$tasklist.="{| class=\"wikitable sortable\"\n";
$tasklist.="! Task !! Disable !! ".(($n==1 || $n==2)?'Approval':'Status')." !! Edit Rate !! Description\n";
$tasklist.=join("\n", map { substr($_,11) } @links)."\n";
$tasklist.="|}\n";
} else {
$tl="<noinclude>\nThis page lists basic information about every task coded for all $mainbot bots, including links to the bot approval and individual shutoff pages.\n\n== Tasks ==\n</noinclude>";
$tasklist.="None at this time.\n";
$acct='! Account !';
}
foreach my $section (sort keys %{$tasks{$pg}}){
my @links=sort @{$tasks{$pg}{$section}};
$section=~s/^(\d+) //;
my $n=$1;
$tl.="\n=== $section ===\n";
if(@links){
$tl.="{| class=\"wikitable sortable\"\n";
$tl.="$acct! Task !! Disable !! {{tlx|bots}} !! ".(($n==1 || $n==2)?'Approval':'Status')." !! Description\n";
$tl.=join("\n", map { substr($_,11) } @links)."\n";
$tl.="|}\n";
} else {
$tl.="None at this time.\n";
}
}
$tasklists{$pg}=$tl;
}
 
$self->{'pages'}={%pages};
$self->{'tasklisttasklists'}=$tasklist\%tasklists;
return $self;
}
 
sub run {
my ($self, $api)=@_;
 
return undef if $self->{'fail'};
 
$self->build_sources() unless defined( $self->{'pages'} );
return undef if $self->{'fail'};
 
my @keys=keys(%{$self->{'pages'}});
if(!my @tlkeys=keys && !exists(%{$self->{'tasklisttasklists'}})){;
my @shutoff=@{$self->warn("Terminating task\n"){'shutoff_pages'}};
my @checkshutoffnotice=@{$self->{'check_shutoff_notice'}};
if(!@keys && !@tlkeys && !@shutoff && !@checkshutoffnotice){
$api->debug(2, "Source uploaded, terminating");
return undef;
}
 
$api->task('SourceUploader', 0, 10, qw/d::Redirects d::Talk/);
$api->read_throttle(0);
$api->edit_throttle(10);
 
my $src='User:'.$api->user.'/source';
 
if($self->{'loadexisting'}){
my %q$iter=$api->iterator(
list => 'allpages',
apprefix => $api->user.'/source/',
apnamespace => '2',
aplimit => 'max'
Line 266 ⟶ 320:
 
my ($k,$v,$k2,$v2,@x);
my while($res;_=$iter->next){
do if(!$_->{'_ok_'}){
$res= $api->querywarn(%q"Failed to retrieve source tree: ".$_->{'error'}."\n");
if($res->{'code'} ne 'success'){
$self->warn("Failed to retrieve source tree: ".$res->{'error'}."\n");
return 300;
}
if(exists$v2=substr($res_->{'query-continuetitle'}, length($src)){;
if(!exists($q{'apfrom'}=$resself->{'query-continuepages'}{'allpages'$v2})){'apfrom'};
$self->{'pages'}{$v2}='';
foreach ( push @{keys, $res->{'query'}{'allpages'}}){v2;
$v2=substr($_->{'title'}, length($src));
if(!exists($self->{'pages'}{$v2})){
$self->{'pages'}{$v2}='';
push @keys, $v2;
}
}
}
} while(exists($res->{'query-continue'}));
 
$self->{'loadexisting'}=0;
Line 294 ⟶ 341:
return $ret if $ret>60;
next if $ret>0;
$ret=redirect_talk($api, $src.$page);
return $ret if $ret;
delete($self->{'pages'}{$page});
}
 
while(@tlkeys){
if(exists($self->{'tasklist'})){
my $page=shift @tlkeys;
my $ret=$self->upload_page($api, 'User:'.$api->user.'/TaskList', $self->{'tasklist'});
my $text=$self->{'tasklists'}{$page};
my $ret=$self->upload_page($api, 'User:'.$api->user."/$page", $text);
return $ret if $ret>60;
next if $ret>0;
delete$ret=redirect_talk($self->{api, 'tasklistUser:'}.$api->user."/$page");
return $ret if $ret;
delete($self->{'tasklists'}{$page});
}
 
while(@shutoff){
my $page=shift @shutoff;
my $tok=$api->edittoken($page);
if($tok->{'code'} eq 'shutoff'){
$api->warn("Task disabled: ".$tok->{'content'}."\n");
return 300;
}
return 60 if($tok->{'code'} ne 'success');
if(exists($tok->{'missing'})){
my $r=$api->edit($tok, '{{subst:void}}', 'Creating empty shutoff page to avoid redlinks in summaries and to avoid confusing people', 1, 1);
if($r->{'code'} ne 'success'){
$api->warn("Write error for $page: ".$r->{'error'}."\n");
return 60;
} else {
$api->log("Created $page");
}
}
my $ret=redirect_talk($api, $page);
return $ret if $ret;
$self->{'shutoff_pages'}=[@shutoff];
}
 
my %x=map { $_=>1 } @checkshutoffnotice;
@checkshutoffnotice=keys %x;
$self->{'check_shutoff_notice'}=[@checkshutoffnotice];
while(@checkshutoffnotice){
my $bot=shift @checkshutoffnotice;
 
my $res=$api->rawpage("Template:Editnotices/Group/User:$bot");
if($res->{'code'} eq 'shutoff'){
$api->warn("Task disabled: ".$res->{'content'}."\n");
return 300;
}
$res->{'code'}='success' if($res->{'code'} eq 'httperror' && $res->{'httpcode'}==404);
if($res->{'code'} ne 'success'){
$api->warn("Could not load Template:Editnotices/Group/User:$bot: ".$res->{'error'}."\n");
return 60;
}
$res=$res->{'content'}//'';
unless($res=~/\{\{Editnotice subpages\|shutoff\|on base=no\}\}/){
$api->whine("Please fix [[Template:Editnotices/Group/User:$bot]]", "To display the proper editnotice on shutoff pages for the AnomieBOT bot [[User:$bot]], please ensure that [[Template:Editnotices/Group/User:$bot]] contains <code><nowiki>{{Editnotice subpages|shutoff|on base=no}}</nowiki></code>. Thanks.");
}
 
$res=$api->rawpage("Template:Editnotices/Group/User:$bot/shutoff");
if($res->{'code'} eq 'shutoff'){
$api->warn("Task disabled: ".$res->{'content'}."\n");
return 300;
}
$res->{'code'}='success' if($res->{'code'} eq 'httperror' && $res->{'httpcode'}==404);
if($res->{'code'} ne 'success'){
$api->warn("Could not load Template:Editnotices/Group/User:$bot/shutoff: ".$res->{'error'}."\n");
return 60;
}
$res=$res->{'content'}//'';
$res=~s/\s*$//;
my $txt="{{#ifeq:{{PAGESIZE:{{FULLPAGENAME}}}}|0|\n{{ombox\n| type = delete\n| image = [[File:Shutdown button.svg|40px|link={{fullurl:Special:Block|wpTarget=".uri_escape($bot)."&wpExpiry=indefinite&wpHardBlock=1&wpAutoBlock=0&wpCreateAccount=0&wpReason=other&wpReason-other=Bot%20malfunctioning:%20}}|Emergency block button]]\n| text = To disable the task {{SUBPAGENAME}}, enter your reasoning in this page. If the task is currently running, the bot may make one more edit before noticing that it has been shut off, but rest assured it ''will'' notice.\n}}\n|{{ombox\n| text = To re-enable the task, blank the page.\n}}\n}}";
unless($res=~/(?:^|\n)\Q$txt\E(?:$|\n)/){
$api->whine("Please fix [[Template:Editnotices/Group/User:$bot/shutoff]]", "To display the proper editnotice on shutoff pages for the AnomieBOT bot [[User:$bot]], please ensure that [[Template:Editnotices/Group/User:$bot/shutoff]] contains the following text:\n<pre>$txt</pre>\nThanks.");
}
 
$self->{'check_shutoff_notice'}=[@checkshutoffnotice];
}
 
Line 315 ⟶ 431:
my $tok=$api->edittoken($page);
if($tok->{'code'} eq 'shutoff'){
$selfapi->warn("Task disabled: ".$tok->{'content'}."\n");
return 300;
}
Line 322 ⟶ 438:
$text=~s/\x02BOT\x03/$bot/go;
$text=~s/\s+$//o;
$tok->{'revisions'}[0]{'slots'}{'main'}{'*'}=~s/\s+$//o if !exists($tok->{'missing'});
if(exists($tok->{'missing'}) ||
$tok->{'revisions'}[0]{'*slots'}{'main'}{'contentmodel'} ne $text){'wikitext' ||
my $r=$api->edit($tok, $text, $self->{'summaryrevisions'}, [0,]{'slots'}{'main'}{'*'} 1ne $text);{
my $r=$api->edit($tok, $text, $self->{'summary'}, 0, 1, contentmodel => 'wikitext' );
if($r->{'code'} ne 'success'){
$selfapi->warn("Write error for $page: ".$r->{'error'}."\n");
return 60;
} else {
$selfapi->warnlog("Updated $page\n");
}
} else {
$selfapi->warndebug(2, "No update needed for $page\n");
}
return 0;
}
 
sub redirect_talk {
my $api=shift;
my $page=shift;
 
my $tpage=$page;
$tpage=~s/^User:/User talk:/;
my $txt=$tpage;
$txt=~s!^(User talk:[^/]+)/.*!$1!;
my $r={$api->resolve_redirects($txt)};
if(exists($r->{''})){
$api->warn("Error fetching redirects for $page: ".$r->{''}{'error'}."\n");
return 60;
}
$txt=$r->{$txt};
$txt="#REDIRECT [[$txt]]\n\nThis page is unwatched. Please direct comments to [[$txt]].";
my $tok=$api->edittoken($tpage, EditRedir=>1, NoExclusion=>1);
if($tok->{'code'} eq 'shutoff'){
$api->warn("Task disabled: ".$tok->{'content'}."\n");
return 300;
}
return 60 if($tok->{'code'} ne 'success');
if(($tok->{'revisions'}[0]{'slots'}{'main'}{'*'} // '') ne $txt){
my $r=$api->edit($tok, $txt, 'Redirect useless unwatched talk page to someplace useful.', 1, 1);
if($r->{'code'} ne 'success'){
$api->warn("Write error for $tpage: ".$r->{'error'}."\n");
return 60;
} else {
$api->log("Created $tpage");
}
}
return undef;
}
 
Line 343 ⟶ 493:
 
my $t=$file; $t=~s{^.*/}{}; $t=~s/\.pm$//;
return '99 Invalid metadata (no "Bot")' unless( exists($metadata{'taskbot'}) && $metadata{'task'} eq $t);
return '99 Invalid metadata (no/bad "Task")' unless (exists($metadata{'brfatask'}) && $metadata{'task'} eq $t);
return '99 Invalid metadata (no "BRFA")' unless exists($metadata{'statusbrfa'});
return '99 Invalid metadata (no "Status")' unless exists($metadata{'ratestatus'});
return '99 Invalid metadata (no/bad "Created")' unless(exists($metadata{'created'}) && $metadata{'created'}=~/^\d{4}-\d\d-\d\d$/);
if(exists($metadata{'+brfa'})){
return '99 Invalid metadata (no "+Status")' unless exists($metadata{'+status'});
return '99 Invalid metadata (mismatched +BRFA/+Status)' if scalar(@{$metadata{'+status'}}) != scalar(@{$metadata{'+brfa'}});
}
if(exists($metadata{'ondemand'})){
return '99 Invalid metadata (bad "OnDemand")' unless($metadata{'ondemand'} eq 'true' || $metadata{'ondemand'} eq 'false');
}
if(exists($metadata{'shutoff'})){
return '99 Invalid metadata (bad "Shutoff")' unless($metadata{'shutoff'} eq 'true' || $metadata{'shutoff'} eq 'false');
}
if(exists($metadata{'exclusion'})){
return '99 Invalid metadata (bad "Exclusion")' unless($metadata{'exclusion'} eq 'true' || $metadata{'exclusion'} eq 'false');
}
my $status=$metadata{'status'};
 
AnomieBOT::TaskAPI::load($file);
my $task='tasks::'.$metadata{'task'};
return '99 Invalid metadata (missing "approved" method)' unless $task->can('approved');
 
my $botnum=$task->approved;
Line 370 ⟶ 526:
return "01 Current" if $botnum>0;
return "02 On demand" if(exists($metadata{'ondemand'}) && $metadata{'ondemand'} eq 'true');
return '99 Invalid metadata ("Status: Approved" but not running or OnDemand)';
}
return '99"03 InvalidIn metadata'trial" if($status=~/^In trial/ && $botnum>0);
return "03'99 InInvalid development"metadata if($statusRunning eqwith 'Codinga non-running Status)' ||if $status eq 'On hold')botnum>0;
return "0504 PastIn development" if ($status=~/^Completed \d{4}-\d{2}-\d{2}eq 'Coding' || $/status eq 'On hold');
return '99"06 Invalid metadata'Past" if( $metadatastatus=~/^(?:Completed|Inactive) \d{'brfa'4} eq 'N-\d{2}-\d{2}$/A');
return "04'99 AwaitingInvalid metadata (bad approval"BRFA")' if($statusmetadata{'brfa'} eq 'BRFAN/A');
return "0605 WithdrawnAwaiting approval" if ($status eq 'WithdrawnBRFA');
return "07 RejectedWithdrawn" if $status=~/^Rejected \d{4}-\d{2}-\d{2}$/eq 'Withdrawn';
return '99"08 InvalidRejected" metadata'if $status=~/^Rejected \d{4}-\d{2}-\d{2}$/;
return '99 Invalid metadata (no match for Status)';
}
 
1;
 
</syntaxhighlight>
</source>
Retrieved from "https://en.wikipedia.org/wiki/User:AnomieBOT/source/tasks/SourceUploader.pm"