#!/usr/bin/perl -s
sub Help {
    print <<HELP;
Synopsis:
    Given that the Reuters-21578 data are in a directory with each document 
    saved in a '.htm' file, extract the title and body of each document 
    successively and save them into a file (with the same name but '.txt' 
    extension) in another directory. See attached texts for example.

Examples:
  perl -s GetTitleBody.pl /test-html /test-txt > test-log.txt
    => Extract testing documents, whose NEWID ranges from 14826 to 21576.
    => Number of testing documents is 3019.
  perl -s GetTitleBody.pl /train-html /train-txt > train-log.txt
    => Extract training documents, whose NEWID ranges from 1 to 14818.
    => Number of training documents is 7770.

Author: Yuen-Hsien Tseng
Date: September 10, 2002
HELP
exit;
}
    &Help() if $help or $h or @ARGV < 2;
    ($Dir, $OutDir) = (@ARGV);
    if (not -d $OutDir) { mkdir($OutDir, 0755) or die "$!"; }
    opendir(DIR, $Dir) or die "Cannot open dir:$Dir, $!";
    @DIR = readdir(DIR) or warn "$!";
    undef $/; # to get the whole file content in one read
    foreach $f (@DIR) {
	next if $f =~ /^\./;

	open F, "$Dir/$f" or warn "Cannot open file:'$Dir/$f', $!";
	$L = <F>; close(F); $T = $B = '';
	if ($L =~ m#<TEXT[^>]*>((.|\s)+)</TEXT>#i) { $L = $1; }
	if ($L =~ m#<TITLE>((.|\s)+)</TITLE>#i) { $T = $1; $T =~ s/&..;//g;}
	if ($L =~ m#<BODY>((.|\s)+)</BODY>#i) { 
	    $B = $1; $B =~ s/&..;//g; # delete HTML special tags: &#3;
	    $B =~ s/REUTER\s*$//i; # delete last 'Reuter' mark
	}
	$ff = $f; $ff =~ s/\.htm/.txt/;
	if ($T eq '' and $B eq '') {
	    $B = $L; $B =~ s/&..;//g; 
	    $B =~ s/Reuter\s*$//i; # delete last 'Reuter' mark
	    print "$ff has no title and body\n" if $B eq '';
	}
	print "$ff has no title\n" if $T eq '';
	print "$ff has no body\n" if $B eq '';
	open F, ">$OutDir/$ff" or warn "Cannot open file for writing, $!";
	print F "$T\n\n$B";
	close(F);
    }


=Below is the example of an HTML file and an extracted TEXT file
================= The HTML file ================================
<REUTERS TOPICS="YES" LEWISSPLIT="TRAIN" CGISPLIT="TRAINING-SET" OLDID="15523" NEWID="10002">
<DATE>26-MAR-1987 12:13:39.63</DATE>
<TOPICS><D>earn</D></TOPICS>
<PLACES><D>usa</D></PLACES>
<PEOPLE></PEOPLE>
<ORGS></ORGS>
<EXCHANGES></EXCHANGES>
<COMPANIES></COMPANIES>
<UNKNOWN> 
&#5;&#5;&#5;E F
&#22;&#22;&#1;f1730&#31;reute
d f BC-island-telephone   03-26 0038</UNKNOWN>
<TEXT>&#2;
<TITLE>ISLAND TELEPHONE SHARE SPLIT APPROVED</TITLE>
<DATELINE>    CHARLOTTETOWN, Prince Edward Island, March 26 - </DATELINE><BODY>&lt;Island
Telephone Co Ltd> said the previously announced two-for-one
common share split was approved by shareholders at the annual
meeting.
 Reuter
&#3;</BODY></TEXT>
</REUTERS>

=================== The extracted TEXT file ======================
ISLAND TELEPHONE SHARE SPLIT APPROVED

Island
Telephone Co Ltd> said the previously announced two-for-one
common share split was approved by shareholders at the annual
meeting.

=cut
