hschumann2/TempleOS-Source-Code
0847
1 2#define DOWNLOAD_FILE1 "TempleOSCD.ISO"3#define DOWNLOAD_FILE1_SIZE 170000004#define DOWNLOAD_FILE2 "TempleOSUltra.ISO"5#define DOWNLOAD_FILE2_SIZE 19000006#define DOWNLOAD_FILE3 "TempleOSCDRS.ISO"7#define DOWNLOAD_FILE3_SIZE 160000008#define DOWNLOAD_FILE4 "TempleOSUltraRS.ISO"9#define DOWNLOAD_FILE4_SIZE 160000010 11#define HOURS_MAX (24*3)12 13class LogStruct14{15 LogStruct *next,*last;16 LogStruct *ip_num_left,*ip_num_right;17 U32 ip_num,code;18 I64 size;19 U8 *file,*link;20 CDate datetime;21};22 23class LinkStruct24{25 LinkStruct *left,*right;26 U8 *link,*file;27 I64 cnt;28};29 30class BlockedStruct31{32 BlockedStruct *next,*last;33 U32 ip_num;34};35 36U0 LogStructDel(LogStruct *tmplg)37{38 Free(tmplg->file);39 Free(tmplg->link);40 Free(tmplg);41}42 43U0 PrsSingleLogFile(LogStruct *head,U8 *name,CDate *_start,CDate *_end)44{45 CDoc *doc=DocRead(name,46 DOCF_PLAIN_TEXT_TABS|DOCF_DBL_DOLLARS|DOCF_NO_CURSOR);47 CDocEntry *doc_e=doc->head.next;48 U8 *src,*src2,*mon_lst=Define("ST_MONTHS");49 LogStruct *tmplg;50 CDateStruct ds;51 I64 i;52 "%$Q\n",name;53 while (doc_e!=doc) {54 if (doc_e->type_u8==DOCT_TEXT) {55 tmplg=CAlloc(sizeof(LogStruct));56 try {57 src=doc_e->tag;58 tmplg->ip_num.u8[3]=Str2I64(src,10,&src);59 if (*src++!='.') throw;60 tmplg->ip_num.u8[2]=Str2I64(src,10,&src);61 if (*src++!='.') throw;62 tmplg->ip_num.u8[1]=Str2I64(src,10,&src);63 if (*src++!='.') throw;64 tmplg->ip_num.u8[0]=Str2I64(src,10,&src);65 66 do if (!*src) throw;67 while (*src++!='[');68 MemSet(&ds,0,sizeof(CDateStruct));69 ds.day_of_mon=Str2I64(src,10,&src);70 if (*src++!='/') throw;71 src2=src;72 do if (!*src2) throw;73 while (*src2++!='/');74 * --src2=0;75 ds.mon=1+LstMatch(src,mon_lst,LMF_IGNORE_CASE);76 src=++src2;77 ds.year=Str2I64(src,10,&src);78 if (*src++!=':') throw;79 ds.hour=Str2I64(src,10,&src);80 if (*src++!=':') throw;81 ds.min=Str2I64(src,10,&src);82 if (*src++!=':') throw;83 ds.sec=Str2I64(src,10,&src);84 tmplg->datetime=Struct2Date(&ds);85 if (*src++!=CH_SPACE) throw;86 i=Str2I64(src,10,&src);87 tmplg->datetime-=(i/100+i%100/60.0)*CDATE_FREQ*60*60;88 89 if (tmplg->datetime<*_start) *_start=tmplg->datetime;90 if (tmplg->datetime>*_end) *_end =tmplg->datetime;91 92 do if (!*src) throw;93 while (*src++!=']');94 95 if (*src++!=CH_SPACE) throw;96 if (*src++!='\"') throw;97 if (!StrNCmp(src,"GET ",4)) {98 src2=src+=4;99 do if (!*src2) throw;100 while (*src2++!=CH_SPACE);101 * --src2=0;102 tmplg->file=StrNew(src);103 src=++src2;104 105 do if (!*src) throw;106 while (*src++!='\"');107 108 tmplg->code=Str2I64(src,10,&src);109 if (*src++!=CH_SPACE) throw;110 tmplg->size=Str2I64(src,10,&src);111 if (*src++!=CH_SPACE) throw;112 113 if (*src++!='\"') throw;114 src2=src;115 do if (!*src2) throw;116 while (*src2++!='\"');117 * --src2=0;118 tmplg->link=StrNew(src);119 src=++src2;120 121 QueIns(tmplg,head->last);122 } else if (!StrNCmp(src,"HEAD ",5)) {123 LogStructDel(tmplg);124 } else125 throw;126 } catch {127 Fs->catch_except=TRUE;128 "%$Q\n",doc_e->tag;129 LogStructDel(tmplg);130 }131 }132 doc_e=doc_e->next;133 }134 DocDel(doc);135}136 137LogStruct *PrsLogFiles(U8 *files_find_mask,CDate *_start,CDate *_end)138{139 LogStruct *head=CAlloc(sizeof(LogStruct));140 CDirEntry *tmpde=FilesFind(files_find_mask),*tmpde1=tmpde;141 QueInit(head);142 while (tmpde) {143 PrsSingleLogFile(head,tmpde->full_name,_start,_end);144 tmpde=tmpde->next;145 }146 DirTreeDel(tmpde1);147 return head;148}149 150U0 LogLstDel(LogStruct *head)151{152 LogStruct *tmplg=head->next,*tmplg1;153 while (tmplg!=head) {154 tmplg1=tmplg->next;155 LogStructDel(tmplg);156 tmplg=tmplg1;157 }158}159 160U0 BlockedStructAdd(BlockedStruct *head,U32 ip_num)161{162 BlockedStruct *tmpb=CAlloc(sizeof(BlockedStruct));163 tmpb->ip_num=ip_num;164 QueIns(tmpb,head->last);165}166 167Bool IsBlocked(BlockedStruct *head,U32 ip_num)168{169 BlockedStruct *tmpb=head->next;170 while (tmpb!=head) {171 if (tmpb->ip_num==ip_num)172 return TRUE;173 tmpb=tmpb->next;174 }175 return FALSE;176}177 178U0 BlockIPNuip(LogStruct *head)179{180 BlockedStruct blocked_head;181 LogStruct *tmplg=head->next,*tmplg1;182 183 QueInit(&blocked_head);184 185 BlockedStructAdd(&blocked_head,70<<24+173<<16+110<<8+214);186 BlockedStructAdd(&blocked_head,68<<24+224<<16+130<<8+96);187 BlockedStructAdd(&blocked_head,68<<24+96<<16+100<<8+126);188 BlockedStructAdd(&blocked_head,68<<24+96<<16+110<<8+213);189 BlockedStructAdd(&blocked_head,68<<24+96<<16+242<<8+108);190 BlockedStructAdd(&blocked_head,68<<24+227<<16+61<<8+11);191 BlockedStructAdd(&blocked_head,68<<24+227<<16+61<<8+139);192 193 //pass 1: collect robot lst194 while (tmplg!=head) {195 if (StrIMatch("ROBOT",tmplg->file) &&196 !IsBlocked(&blocked_head,tmplg->ip_num))197 BlockedStructAdd(&blocked_head,tmplg->ip_num);198 tmplg=tmplg->next;199 }200 201 //pass 2: removed blocked ip_nuip202 tmplg=head->next;203 while (tmplg!=head) {204 tmplg1=tmplg->next;205 if (IsBlocked(&blocked_head,tmplg->ip_num)) {206 QueRem(tmplg);207 LogStructDel(tmplg);208 }209 tmplg=tmplg1;210 }211 212 QueDel(&blocked_head);213}214 215Bool IsDownLoad(LogStruct *tmplg)216{217 if (StrMatch(DOWNLOAD_FILE1,tmplg->file)&&tmplg->size>=218 DOWNLOAD_FILE1_SIZE ||219 StrMatch(DOWNLOAD_FILE2,tmplg->file)&&tmplg->size>=220 DOWNLOAD_FILE2_SIZE ||221 StrMatch(DOWNLOAD_FILE3,tmplg->file)&&tmplg->size>=222 DOWNLOAD_FILE3_SIZE ||223 StrMatch(DOWNLOAD_FILE4,tmplg->file)&&tmplg->size>=224 DOWNLOAD_FILE4_SIZE)225 return TRUE;226 else227 return FALSE;228}229 230Bool IsIndex(LogStruct *tmplg)231{232 if (!StrCmp(tmplg->file,"/index.html") ||233 !StrNCmp(tmplg->file+StrLen(tmplg->file)-14,"/TempleOS.html",14) ||234 !StrCmp(tmplg->file,"/"))235 return TRUE;236 else237 return FALSE;238}239 240Bool IPNumTreeAdd(LogStruct **_head,LogStruct *tmplg)241{242 LogStruct *head;243 if (UnusedStk<0x200) {244 PrintErr("Stk Overflow");245 throw;246 }247 if (head=*_head) {248 if (tmplg->ip_num==head->ip_num)249 return TRUE;250 else if (tmplg->ip_num<head->ip_num)251 return IPNumTreeAdd(&head->ip_num_left,tmplg);252 else253 return IPNumTreeAdd(&head->ip_num_right,tmplg);254 } else {255 tmplg->ip_num_left=NULL;256 tmplg->ip_num_right=NULL;257 *_head=tmplg;258 return FALSE;259 }260}261 262U0 LinkTreeAdd(LinkStruct **_root,LogStruct *tmplg)263{264 I64 i;265 LinkStruct *root,*tmplk;266 if (UnusedStk<0x200) {267 PrintErr("Stk Overflow");268 throw;269 }270 if (root=*_root) {271 if (!(i=StrCmp(tmplg->link,root->link)))272 root->cnt++;273 else if (i<0)274 LinkTreeAdd(&root->left,tmplg);275 else276 LinkTreeAdd(&root->right,tmplg);277 } else {278 tmplk=CAlloc(sizeof(LinkStruct));279 tmplk->link=tmplg->link;280 tmplk->cnt=1;281 *_root=tmplk;282 }283}284 285U0 FileTreeAdd(LinkStruct **_root,LogStruct *tmplg)286{287 I64 i;288 LinkStruct *root,*tmplk;289 if (UnusedStk<0x200) {290 PrintErr("Stk Overflow");291 throw;292 }293 if (root=*_root) {294 if (!(i=StrCmp(tmplg->file,root->file)))295 root->cnt++;296 else if (i<0)297 FileTreeAdd(&root->left,tmplg);298 else299 FileTreeAdd(&root->right,tmplg);300 } else {301 tmplk=CAlloc(sizeof(LinkStruct));302 tmplk->file=tmplg->file;303 tmplk->cnt=1;304 *_root=tmplk;305 }306}307 308U0 LinkTreeDel(LinkStruct *root)309{310 if (root) {311 LinkTreeDel(root->left);312 LinkTreeDel(root->right);313 Free(root);314 }315}316 317U0 LinkTreeTraverse(LinkStruct *root)318{319 if (root) {320 LinkTreeTraverse(root->left);321 "%3d:%$Q\n",root->cnt,root->link;322 LinkTreeTraverse(root->right);323 }324}325 326U0 FileTreeDel(LinkStruct *root)327{328 if (root) {329 FileTreeDel(root->left);330 FileTreeDel(root->right);331 Free(root);332 }333}334 335U0 FileTreeTraverse(LinkStruct *root)336{337 if (root) {338 FileTreeTraverse(root->left);339 "%3d:%$Q\n",root->cnt,root->file;340 FileTreeTraverse(root->right);341 }342}343 344U0 DownLoadRep(LogStruct *head,CDate start,CDate end)345{346 I64 i,j,cnt,dups,347 hours_start,hours_end,*hour_cnts,*dup_cnts,348 days_start,days_end,*day_cnts,*day_dup_cnts;349 LogStruct *tmplg=head->next,*dup_head=NULL;350 LinkStruct *link_root=NULL;351 CDateStruct ds;352 353 i=start*24;hours_start=i.u32[1];354 i=end *24;hours_end =i.u32[1];355 356 days_start=(start+local_time_offset)>>32;357 days_end =(end+local_time_offset)>>32;358 359 hour_cnts=CAlloc((hours_end-hours_start+1)*sizeof(I64));360 dup_cnts =CAlloc((hours_end-hours_start+1)*sizeof(I64));361 day_cnts =CAlloc((days_end-days_start+1)*sizeof(I64));362 day_dup_cnts=CAlloc((days_end-days_start+1)*sizeof(I64));363 dups=cnt=0;364 while (tmplg!=head) {365 if (start<=tmplg->datetime<=end && IsDownLoad(tmplg)) {366 i=tmplg->datetime*24;367 hour_cnts[i.u32[1]-hours_start]++;368 day_cnts[(tmplg->datetime+local_time_offset)>>32-days_start]++;369 cnt++;370 if (IPNumTreeAdd(&dup_head,tmplg)) {371 day_dup_cnts[(tmplg->datetime+local_time_offset)>>32-days_start]++;372 dup_cnts[i.u32[1]-hours_start]++;373 dups++;374 }375 LinkTreeAdd(&link_root,tmplg);376 }377 tmplg=tmplg->next;378 }379 380 "\n\nDownloads of /TempleOSCD.ISO\n";381 for (i=start;i<=end;i+=1<<32)382 "%D Dups:%5d Total:%5d Uniques:%5d\n",i,383 day_dup_cnts[(i+local_time_offset)>>32-days_start],384 day_cnts[(i+local_time_offset)>>32-days_start],385 day_cnts[(i+local_time_offset)>>32-days_start]-386 day_dup_cnts[(i+local_time_offset)>>32-days_start];387 388 "\n\nDownloads of /TempleOSCD.ISO\n"389 "'-' is a dup. '+' is not a dup.\n";390 if (hours_end-hours_start>=HOURS_MAX)391 i=hours_end-HOURS_MAX+1;392 else393 i=hours_start;394 for (;i<=hours_end;i++) {395 Date2Struct(&ds,i<<32/24+local_time_offset);396 "%D %02d: ",i<<32/24,ds.hour;397 for (j=0;j<dup_cnts[i-hours_start];j++)398 '-';399 for (;j<hour_cnts[i-hours_start];j++)400 '+';401 '\n';402 }403 "Total:%d Dups:%d Uniques:%d\n",cnt,dups,cnt-dups;404 405 "\n\nDownloads of /TempleOSCD.ISO\n";406 LinkTreeTraverse(link_root);407 '\n';408 409 LinkTreeDel(link_root);410 Free(hour_cnts);411 Free(dup_cnts);412 Free(day_cnts);413 Free(day_dup_cnts);414}415 416U0 FileRep(LogStruct *head,CDate start,CDate end)417{418 LogStruct *tmplg=head->next;419 LinkStruct *file_root=NULL;420 while (tmplg!=head) {421 if (start<=tmplg->datetime<=end)422 FileTreeAdd(&file_root,tmplg);423 tmplg=tmplg->next;424 }425 "\n\nFile Hits\n";426 FileTreeTraverse(file_root);427 '\n';428 FileTreeDel(file_root);429}430 431U0 IndexRep(LogStruct *head,CDate start,CDate end)432{433 I64 i,j,cnt,dups,434 hours_start,hours_end,*hour_cnts,*dup_cnts,435 days_start,days_end,*day_cnts,*day_dup_cnts;436 LogStruct *tmplg=head->next,*dup_head=NULL;437 LinkStruct *link_root=NULL;438 CDateStruct ds;439 440 i=start*24;hours_start=i.u32[1];441 i=end *24;hours_end =i.u32[1];442 443 days_start=(start+local_time_offset)>>32;444 days_end =(end+local_time_offset)>>32;445 446 hour_cnts=CAlloc((hours_end-hours_start+1)*sizeof(I64));447 dup_cnts =CAlloc((hours_end-hours_start+1)*sizeof(I64));448 day_cnts =CAlloc((days_end-days_start+1)*sizeof(I64));449 day_dup_cnts=CAlloc((days_end-days_start+1)*sizeof(I64));450 dups=cnt=0;451 while (tmplg!=head) {452 if (start<=tmplg->datetime<=end && IsIndex(tmplg)) {453 i=tmplg->datetime*24;454 hour_cnts[i.u32[1]-hours_start]++;455 day_cnts[(tmplg->datetime+local_time_offset)>>32-days_start]++;456 cnt++;457 if (IPNumTreeAdd(&dup_head,tmplg)) {458 day_dup_cnts[(tmplg->datetime+local_time_offset)>>32-days_start]++;459 dup_cnts[i.u32[1]-hours_start]++;460 dups++;461 }462 LinkTreeAdd(&link_root,tmplg);463 }464 tmplg=tmplg->next;465 }466 467 "\n\nHits on /index.html\n"468 "'-' is a dup. '+' is not a dup.\n";469 for (i=start;i<=end;i+=1<<32)470 "%D Dups:%5d Total:%5d Uniques:%5d\n",i,471 day_dup_cnts[(i+local_time_offset)>>32-days_start],472 day_cnts[(i+local_time_offset)>>32-days_start],473 day_cnts[(i+local_time_offset)>>32-days_start]-474 day_dup_cnts[(i+local_time_offset)>>32-days_start];475 476 "\n\nHits on /index.html\n";477 if (hours_end-hours_start>=HOURS_MAX)478 i=hours_end-HOURS_MAX+1;479 else480 i=hours_start;481 for (;i<=hours_end;i++) {482 Date2Struct(&ds,i<<32/24+local_time_offset);483 "%D %02d: ",i<<32/24,ds.hour;484 for (j=0;j<dup_cnts[i-hours_start];j++)485 '-';486 for (;j<hour_cnts[i-hours_start];j++)487 '+';488 '\n';489 }490 "Total:%d Dups:%d Uniques:%d\n",cnt,dups,cnt-dups;491 492 "\n\nHits on /index.html\n";493 LinkTreeTraverse(link_root);494 '\n';495 496 LinkTreeDel(link_root);497 Free(hour_cnts);498 Free(dup_cnts);499 Free(day_cnts);500 Free(day_dup_cnts);501}502 503U0 WebLogRep(U8 *mask,U8 *output_filename)504{505 LogStruct *head;506 CDate start=I64_MAX,end=I64_MIN;507 508 DocMax;509 510 head=PrsLogFiles(mask,&start,&end);511 start=GetDate("Start(%D):",start);512 end =GetDate("End (%D):",end);513 BlockIPNuip(head);514 515 DocClear;516 "$WW,0$";517 IndexRep(head,start,end);518 FileRep(head,start,end);519 DownLoadRep(head,start,end);520 521 StrCpy(DocPut->filename.name,output_filename);522 DocWrite(DocPut,TRUE);523 524 "$WW,1$";525 LogLstDel(head);526}527 528#if __CMD_LINE__529Cd(__DIR__);;530WebLogRep("*.LOG*","~/DemoWebLog.DD.Z");531#endif532