rm dict1.txt 

# Go over all files in directory RefData and take the 20 most common words
# in each file. 
for i in `ls -t ../RefData/`
do
    tail -20 ../RefData/$i | awk '{print $1}' >> dict1.txt
done

# Take the 5000 most common words of these
cat dict1.txt | sort | uniq -c | sort -n -r +1 | head -5000 | awk '{print $2}' > dictionary.txt


./make_stat_word.csh dictionary.txt

#return; 

s=1

# Analyze the word statistics ofall the files in the director RefData
# and output to directory Stat
for i in `ls -t ../RefData`
do
    awk -f word_statistics.awk ../RefData/$i > ../Stat/$i.stat
    s=`expr $s + 1`

    echo "Value of s is " $s
#    if [ $s -gt 10 ]
#    then 
#	echo "S value large"
#	exit; 
#    fi
done

ls -t ../Stat/ > stat_filelist.txt
