#!/bin/sh

echo "Now processing file" $1 " --> " $1.ref

a=`wc -w $1 | awk '{print $1}'`

# DON'T PROCESS FILES WITH LESS THAN 100 WORDS (?)
if [ $a -lt 150 ]
then 
    echo "Small File, number of words= " $a "... exiting"
    return;
fi

# First coversion to lowercase, then 
# Remove the following signs: =0-9 , ? : * [ ] . ( )  ; @ " - / and a single quote
# Note that the first - removes the minus sign. Should not put anything in front of it
cat $1 | awk '{a = tolower($0); gsub(/[-+|=%&#!_{}0-9<>~()\[\]:?,\*\"\.;\$\\@]/," ",a);  gsub("/"," ",a); gsub("'\''"," ",a); print a;}' > tmp3.txt; 

# Create a sorted file with one word per line 
cat tmp3.txt | awk '{for(i=1; i<NF+1; i++) print $i}' > tmp4.txt; #tmp4 contains one word per line

# Replace words ending with lly to ending with l, such as mathematically -> mathematical
# Replace words ending ies to ending y such as galaxies -> galaxy
# Problem: species -> specy but that is ok. 
# Replace words ending hing to ending h as in breathing -> breath, fishing -> fish
# Replace ending nning -> n such as running -> run
# Replace ending ved -> ve, as in served -> serve
cat tmp4.txt | awk '{a=$1; gsub(/lly$/,"l",a); gsub(/ved$/,"ve",a); gsub(/ies$/,"y",a); gsub(/hing$/,"h",a); gsub(/nning$/,"n"); print a;}' > tmp5.txt

# Remove last s by space unless it is a double ss. 
cat tmp5.txt | awk '{a=$1; b=a; gsub(/s$/," ",b); if( a ~ /ss$/ ){b = a} if (a~/us$/){b=a} if(a == "aids") {b = a} if(a ~ /is$/){b=a} print b}' > tmp6.txt; 

# Remove common words
sort tmp6.txt | awk -f remove_words2.awk > tmp7.txt; 

cat tmp7.txt | ./freq.csh | sort -n +1 > $1.ref













