// StringMatch.cpp : Defines the entry point for the console application.
//

// #include "stdafx.h"
#include "stdio.h"
#include "stdlib.h"
#include "string.h"
#include "malloc.h"
#include "ctype.h"
#include "assert.h"

char chFileSeparator = '\\';

char szDic[2560];
char szBufDic[2560];
char szBuf2Dic[2560];
char szBuf3Dic[2560];

typedef char* SZ;

SZ rgszdic[100000];
int cdic;

void ErrorExit()
{
	_flushall();
	exit(1);
}

void WhackWhite(char* sz)
{
	int i = strlen(sz);
	unsigned char* p = (unsigned char*)(sz + i);
	if (*p == 0)
		--p;

	while ((p >= (unsigned char*)sz) && (isspace(*p)))
	{
		*p-- = 0;
	}
}

int mystrcmp(unsigned char* p1, unsigned char* p2)
{
	while ((*p1) && (*p2) && (*p1 == *p2))
	{
		++p1;
		++p2;
	}

	return *p1 - *p2;
}

int strcmpLower(unsigned char* sz1, unsigned char* sz2)
{
	if (!sz1)
		return 1;

	if (!sz2)
		return 1;

	unsigned char ch1 = sz1[0];
	sz1[0] = tolower(ch1);

	unsigned char ch2 = sz2[0];
	sz2[0] = tolower(ch2);

	int iRet = mystrcmp(sz1, sz2);

	sz1[0] = ch1;
	sz2[0] = ch2;

	return iRet;
}

void TwiddleDic()
{
	assert(cdic);
	int i = cdic-1;
	char* szT;

	while ((i) && (0 > strcmpLower((unsigned char*)(rgszdic[i]), (unsigned char*)(rgszdic[i-1]))))
	{
		int iT = strcmpLower((unsigned char*)rgszdic[i], (unsigned char*)rgszdic[i-1]);
		szT = rgszdic[i-1];
		rgszdic[i-1] = rgszdic[i];
		rgszdic[i] = szT;
		--i;
	}
}

void InsertIntoDic(unsigned char* szT)
{
	int i;
	int idic = cdic++;
	int cnt = strlen((char*)szT);
	unsigned char* sz = (unsigned char*) malloc(cnt+1);
	strcpy((char*)sz, (char*)szT);

	for (i=1; i<cnt; ++i)
		sz[i] = tolower(sz[i]);

	rgszdic[idic] = (char*)sz;
	TwiddleDic();
}

int ReadDic(char* szDicPath)
{
	char* pSep = szDicPath+strlen(szDicPath);
	
	while (*pSep != chFileSeparator)
		--pSep;

	int cDirectoryPath = pSep - szDicPath;

	char* pfilename = (char*) malloc(cDirectoryPath + 20);
	strncpy(pfilename, szDicPath, cDirectoryPath+1);
	strcpy(pfilename+cDirectoryPath+1, "GutDicEN.txt");

	// FILE* pfile = fopen(pfilename, "r"); Don't use the deprecated version unless you have to

	FILE* pfile;
	int iret = fopen_s(&pfile, pfilename, "r"); 

	if (iret)
	{
		fprintf(stderr, "Unable to open Dictionary File which needs to be at location: %s\n", pfilename);
		ErrorExit();
	}
	
	while (fgets(szDic, 250, pfile))
	{
		WhackWhite(szDic);
		InsertIntoDic((unsigned char*)szDic);
	}

	fclose(pfile);

	return 0;
}

int IsValidDicWord(char* szIn)
{
	if (!szIn)
		return 0;

	if (szIn[0] == 0)
		return 0;

	assert(cdic);
	strcpy((char*)szBuf3Dic, szIn);
	int cnt = strlen(szIn);
	int stepsize = cdic/2;
	int i = stepsize;
	int iRet;

	while (stepsize > 1)
	{
		stepsize = stepsize / 2;
		iRet = strcmpLower((unsigned char*)szBuf3Dic, (unsigned char*)(rgszdic[i]));

		if (iRet == 0)
			break;
		else if (iRet < 0)
			i = i - stepsize;
		else
			i = i + stepsize;
	}

	while (i && ((iRet = strcmpLower((unsigned char*)szBuf3Dic, (unsigned char*)(rgszdic[i]))) < 0))
	{
		--i;
	}

	while ((i < cdic) && ((iRet = strcmpLower((unsigned char*)szBuf3Dic, (unsigned char*)(rgszdic[i]))) > 0))
	{
		++i;
	}

	if (0 != strcmpLower((unsigned char*)szBuf3Dic, (unsigned char*)(rgszdic[i])))
		return 0;

	if (isupper(((unsigned char**)rgszdic)[i][0]))
	{
		if (isupper(((unsigned char*)szBuf3Dic)[0]))
			return 1;
		else
			return 0;
	}

	return 1;
}

SZ rgszsub[] = 
{
	" ;", ";",
	" ,", ",",
	" :", ":",
	" ?", "?",
	" !", "!",
	" n't", "n't",
	" 's", "'s",
	" 'd", "'d",
	" 're", "'re",
	" - ", "-",
	" -- ", "--",
	"''", "\""
};

int fTightenPunc(char* sz)
{
	int fReturn = 0;
	int i;
	int ioffset = 0;
	char* p1 = 0;
	char* p2 = 0;
	SZ sz1 = 0;
	SZ sz2 = 0;
	int c = (sizeof(rgszsub));
	c /= (sizeof(SZ));

	for (i = 0; i < c; i += 2)
	{
		sz1 = rgszsub[i];
		sz2 = rgszsub[i+1];

		p1 = strstr(sz, sz1);

		if (p1 == 0)
			continue;

		fReturn = 1;

		strcpy(p1, sz2);

		p1 += strlen(sz2);

		ioffset = strlen(sz1) - strlen(sz2);

		p2 = p1 + ioffset;

		while (*p2)
		{
			*p1++ = *p2++;
		}

		*p1 = 0;
	}

	return fReturn;
}

void SmartDehyphen(char* sz1, char* sz2)
{
	int i = 0;
	unsigned char* p2 = (unsigned char*)sz2;
	unsigned char* p0 = (unsigned char*)sz2;

	if (!isalnum(*p2))
		return;

	unsigned char* pprev = (unsigned char*)(sz1 + strlen(sz1)-2);

	while (isalnum(*pprev))
		--pprev;

	++pprev;

	while (isalnum(*p2))
		++p2;

	while (ispunct(*p2))
		++p2;

	i = p2 - (unsigned char*)sz2;

	int cnt = strlen(sz1);

	strncpy(sz1+cnt, sz2, i);
	sz1[cnt+i] = 0;

	if (isspace(*p2))
		++p2;

	while(*p2)
	{
		*p0++ = *p2++;
	}
	*p0 = 0;

	strcpy(szBuf2Dic, (char*)pprev);

	unsigned char* p = (unsigned char*)(szBuf2Dic + strlen(szBuf2Dic) - 1);

	if (*p == '-')
		--p;

	while (*p != '-')
		--p;

	while (*p)
	{
		p[0] = p[1];
		++p;
	}

	p = (unsigned char*)szBuf2Dic;

	while (*p && (!ispunct(*p)))
	{
		++p;
	}

	unsigned char chPunc = *p;
	*p = 0;

	if (IsValidDicWord(szBuf2Dic))
	{
		*p = chPunc;
		strcpy((char*)pprev, szBuf2Dic);
	}
}

void MarkWord(unsigned char* sz, int cnt)
{
	unsigned char* p = sz + strlen((char*) sz);
	unsigned char* p2;

	if (p == (sz+cnt))
	{
		*p++ = ' ';
		*p = 0;
		--p;
		p2 = p + 6;
		*p2 = 0;
	}

	p2 = p + 5;
	unsigned char* p0 = sz + cnt;

	while (p > p0)
	{
		*p2-- = *p--;
	}

	*p2-- = ' ';
	*p2-- = '}';

	while (p >= sz)
	{
		*p2-- = *p--;
	}

	*p2-- = ' ';
	*p2-- = '{';
	*p2-- = ' ';
}

void MarkSpelling(unsigned char* sz)
{
	unsigned char* p = sz + strlen((char*)sz);
	unsigned char* p2;

	while (p > sz)
	{
		while ((p > sz) && (!isalpha(*p)))
			--p;

		p2 = p+1;

		if ((p[-1] == '\'') && (p[0] == 's') && (isalpha(p[-2])))
			p -= 2;
		else if ((p[-1] == '\'') && (p[0] == 't') && (isalpha(p[-2])))
			p -= 2;

		if (!isalpha(*p))
			continue;

		while ((p >= sz) && (isalpha(*p)))
			--p;

		if ((p[0] == '\'') && (p[-1] == 'o'))
			p -= 2;

		++p;

		int cnt = p2 - p;

		if (cnt > 0)
		{
			unsigned char chT = *p2;
			*p2 = 0;

			if (IsValidDicWord((char*)p))
			{
				*p2 = chT;
				--p;
				continue;
			}

			*p2 = chT;
			MarkWord(p, cnt);
			--p;
		}



	}
}

int icnt = 0;

int PrepDic(int argc, char* argv[])
{
	int iTest = 0;
	int fInit = 1;
	int fHyphen = 0;
	int fMarkSpelling = 0;
	int fHyphenPrev = 0;
	int fDicBad = ReadDic(argv[0]);
	assert(fDicBad == 0);
	return 0;
}


#define D(x, y) d[((maxsize)*(x)) + (y)]

int chBuf1 = 'X';
int chBuf2 = 'X';
char szBuf[2048];
char szBufW[2048];

struct TOK
{
	unsigned iHash;
	char* szTok;
	char* szWhite;
};

#define EQUALTOKS(x, y) (((x).iHash == (y).iHash) && (0 == strcmp((x).szTok, (y).szTok)))



void PrintTOK(TOK* pTok)
{
	printf("%s | %d \n", pTok->szTok, pTok->iHash);
}

void PrintTOKS(TOK* pTok1, TOK* pTok2)
{
	printf("%s | %d |", pTok1->szTok, pTok1->iHash);
	printf("%s | %d |", pTok2->szTok, pTok2->iHash);
}

void InitTOK(TOK* pTok)
{
	pTok->iHash = 0;
	pTok->szTok = 0;
	pTok->szWhite = 0;
}

unsigned HASH(char* p)
{
	unsigned ihash = 0;
	while (*p)
	{
		ihash = ((unsigned)(*p)) ^ (ihash << 7) ^ (ihash >> (32 - 7));
		++p;
	}
	return ihash;
}

void GetTok(int* pch, TOK* pTok, FILE* file)
{
	char* p = szBuf;
	char* pW = szBufW;

	*p = 0;
	*pW = 0;

	if (*pch == 0)
	{
		*pch = fgetc(file);
		while (isspace(*pch) && *pch != EOF)	// flush possible leading whitespace;
		{
			*pch = fgetc(file);
		}
	}
	while (!isspace(*pch) && *pch != EOF)// get non-white token
	{
		*p++ = *pch;
		*p = 0;
		*pch = fgetc(file);
	}
	while (isspace(*pch) && *pch != EOF)// get following white part
	{
		*pW++ = *pch;
		*pW = 0;
		*pch = fgetc(file);
	}

	if (pTok->szTok) free(pTok->szTok);
	if (pTok->szWhite) free(pTok->szWhite);

	pTok->szTok = (char*) malloc(1 + strlen(szBuf));
	strcpy(pTok->szTok, szBuf);

	pTok->szWhite = (char*) malloc(1 + strlen(szBufW));
	strcpy(pTok->szWhite, szBufW);

	pTok->iHash =  HASH(pTok->szTok);

//	PrintTOK(pTok);
}

void GetToks(int* pch, TOK* s, int maxsize, FILE* F1)
{
	++s;
	for (int i = 0; i < maxsize; ++i)
	{
		GetTok(pch, s+i, F1);
	}
}

void FreeTOK(TOK* t)
{
	if (t->szTok)
		free(t->szTok);

	t->szTok = 0;

	if (t->szWhite)
		free(t->szWhite);

	t->szWhite = 0;
}

void ShiftToks(TOK* s, int cTraceback, int maxsize)
{
	++s;
	int i, j;

	for (i = 0; i < cTraceback; ++i)
	{
		FreeTOK(s+i);
	}

	j = cTraceback;

	for (i = 0; i < (maxsize-cTraceback); ++i)
	{
		s[i] = s[j];
		++j;
	}

	for (i = (maxsize-cTraceback); i < maxsize; ++i)
	{
		InitTOK(s+i);
	}
}

void DumpTOKS(TOK* s, TOK* t, int maxsize)
{
	++s;
	++t;
	int i;

	for (i = 0; i < maxsize; ++i)
	{
		PrintTOK(s+i);
	}

	printf("\n");

	for (i = 0; i < maxsize; ++i)
	{
		PrintTOK(t+i);
	}

	fflush(stdout);

//	ErrorExit();
}

short min3(short i, short j, short k)
{
	int t = i;
	if (j < t) t = j;
	if (k < t) t = k;
	return t;
}


void LD(int maxsize, short* d, TOK* s, TOK* t)
{
	int m = maxsize;
	int n = maxsize;
	int i, j, k, iJunk;
	TOK* sT;
	TOK* tT;

	for (i = 0; i <= m; ++i)
		D(i, 0) = i;

	for (j = 0; j <= n; ++j)
		D(0, j) = j;

	for (j = 1; j <= n; ++j)
	{
		tT = &(t[j]);
		for (i = 1; i <= m; ++i)
		{
			sT = &(s[i]);
			if (EQUALTOKS(s[i],t[j]))
			{
				D(i, j) = D(i-1, j-1);
				iJunk = D(i, j);
			}
			else
			{
				k = min3  (
									D(i-1, j) + 1,	// deletion
									D(i, j-1) + 1,	// insertion
									D(i-1, j-1) + 1	// substitution
								);
				D(i, j) = k;
			}
		}
	}
}

void traceback(int maxsize, char* rg, short* d, int m, int n)
{
	char* p = rg;
	char* p2;
	
	int i = m;
	int j = n;

	char cT;

	while ((i > 0) && (j > 0))
	{
		int min;
		cT = 0;

	
		min = D(i-1, j-1);
		cT = 's';	// assume substitute

		if (D(i-1, j) < min)
		{
			min = D(i-1, j);
			cT = 'd';	// assume delete
		}
		else if (D(i, j-1) < min)
		{
			min = D(i, j-1);
			cT = 'i';	// insert
		}

		if (cT == 's' && (D(i-1, j-1) == D(i, j)))
		{		
			cT = 'e';	// emit, actually
		}

		*p++ = cT;
		*p = 0;

		if ((cT == 's') || (cT == 'e'))
		{
			i--;
			j--;
		}
		else if (cT == 'd')
		{
			i--;
		}
		else // t == i
		{
			j--;
		}
	}

	*p-- = 0;
	p2 = rg;

	// reverse string in place

	while (p2 < p)
	{
		cT = *p2;
		*p2 = *p;
		*p = cT;
		++p2;
		--p;
	}
}

int truncateTraceback(int maxsize, char* ptraceback)
{
	char* p;

	int truncatePoint = maxsize / 2;

	int cHits = 0;

	p = ptraceback;

	while (*p)
	{
		if (*p == 'e')
			++cHits;
		++p;
	}

	if (cHits < truncatePoint)
	{
		fprintf(stderr, "Too many errors while matching words -- lost syncronization, aborting -- try running again with a larger -w option value!\n");
		fprintf(stderr, "%s\n", ptraceback);
		ErrorExit();
	}

	p = ptraceback + truncatePoint;

	while (*p != 'e')
		--p;

	*p = 0;

	return p - ptraceback;
}

int strcpycount(char* p, char* sz, char* szWhite)
{
	int i, iW;
	i = strlen(sz);
	iW = strlen(szWhite);
	strcpy(p, sz);
	strcpy(p + i, szWhite);
	return i + iW;
}

int strcpyTok(char* p, TOK* t)
{
	return strcpycount(p, t->szTok, t->szWhite);
}

int strcpycountNoWhite(char* p, char* sz)
{
	int i;
	i = strlen(sz);
	strcpy(p, sz);
	return i;
}

int strcpyTokNoWhite(char* p, TOK* t)
{
	return strcpycountNoWhite(p, t->szTok);
}

int strcpyTokJustWhite(char* p, TOK* t)
{
	return strcpycountNoWhite(p, t->szWhite);
}


int strcpyTokSpace(char* p, TOK* t)
{
	int i = strcpycountNoWhite(p, t->szTok);
	p[i] = ' ';
	return i + 1;
}

char* DicVote(char* pOut, TOK* s, TOK* t)
{
	int i;
	int fWordS = IsValidDicWord(s->szTok);
	int fWordT = IsValidDicWord(t->szTok);

	if (fWordS && fWordT)
		return 0;

	if (!fWordS && !fWordT)
		return 0;

	if (fWordS)
		i = strcpyTok(pOut, s);
	else
		i = strcpyTok(pOut, t);

	return pOut+i;
}

void performEdit(char* pOutT, char* ptraceback, TOK* s, TOK* t, int* pcTraceback1, int* pcTraceback2)
{
	++s;
	++t;
	char c = 0;
	char cT = 0;
	int fOpen = 0;
	TOK* sT = s;
	TOK* tT = t;

	char* pOut = pOutT;

	while (c = *ptraceback++)
	{
		if (c == 'e') // emit
		{
//			assert(0 == strcmp(s->szTok, t->szTok));
			if (fOpen)
			{
				*pOut++ = '}';
				*pOut++ = ' ';
				*pOut = 0;
				fOpen = 0;
			}
			pOut += strcpyTok(pOut, s++);
			*pOut = 0;
			++t;

			cT = 'e';
		}
		else if (c == 's') // substitute
		{
			char* pOutT = DicVote(pOut, t, s);

			if (pOutT)	// then we have voted for a dictionary word
			{
				pOut = pOutT;
				s++;
				t++;
			}
			else
			{
				if (cT == 'e')
				{
					if (fOpen)
					{
						*pOut++ = '}';
						*pOut++ = ' ';
						*pOut = 0;
					}
				
					*pOut++ = '{';
					*pOut++ = ' ';
					*pOut = 0;
					fOpen = 1;
				}

				if (cT != 'd')
					pOut += strcpyTokNoWhite(pOut, t++);
				else
					pOut += strcpyTokNoWhite(pOut, s++);

				*pOut++ = ' ';
				*pOut++ = '|';
				*pOut++ = ' ';
				*pOut = 0;

				if (cT != 'd')
					pOut += strcpyTok(pOut, s++);
				else
					pOut += strcpyTok(pOut, t++);

				*pOut = 0;

				cT = 's';
			}
		}
		else if (c == 'd') // delete
		{
			if (cT == 'e')
			{
				if (fOpen)
				{
					*pOut++ = '}';
					*pOut++ = ' ';
					*pOut = 0;
				}
			
				*pOut++ = '{';
				*pOut++ = ' ';
				*pOut = 0;
				fOpen = 1;
			}

			pOut += strcpyTok(pOut, s++);
			*pOut = 0;

			cT = 'd';
		}
		else if (c == 'i') // insert
		{
			if (cT == 'e')
			{
				if (fOpen)
				{
					*pOut++ = '}';
					*pOut++ = ' ';
					*pOut = 0;
				}
				
				*pOut++ = '{';
				*pOut++ = ' ';
				*pOut = 0;
				fOpen = 1;
			}

			pOut += strcpyTok(pOut, t++);
			*pOut = 0;

			cT = 'i';
		}
	}

	if (fOpen)
	{
		*pOut++ = '}';
		*pOut++ = ' ';
		*pOut = 0;
	}

	*pOut = 0;

	*pcTraceback1 = s - sT;
	*pcTraceback2 = t - tT;
}

void performJustDiffs(char* pOutT, char* ptraceback, TOK* s, TOK* t, int* pcTraceback1, int* pcTraceback2)
{
	++s;
	++t;
	char c = 0;
	char cT = 0;
	int fOpen = 0;
	TOK* sT = s;
	TOK* tT = t;

	char* pOut = pOutT;

	while (c = *ptraceback++)
	{
		if (c == 'e') // emit
		{
//			assert(0 == strcmp(s->szTok, t->szTok));
			if (fOpen)
			{
				*pOut++ = '\n';
				*pOut = 0;
				fOpen = 0;
			}
			s++;
			*pOut = 0;
			++t;

			cT = 'e';
		}
		else if (c == 's') // substitute
		{
			if (cT == 'e')
			{
				if (fOpen)
				{
					*pOut++ = '\n';
					*pOut = 0;
				}
				
				*pOut = 0;
				fOpen = 1;
			}

			if (cT != 'x')
				pOut += strcpyTokSpace(pOut, t++);
			else
				pOut += strcpyTokSpace(pOut, s++);

			*pOut++ = '-';
			*pOut++ = '>';
			*pOut++ = ' ';
			*pOut = 0;

			if (cT != 'x')
				pOut += strcpyTokSpace(pOut, s++);
			else
				pOut += strcpyTokSpace(pOut, t++);

			*pOut++ = '|';
			*pOut++ = ' ';
			*pOut = 0;

			*pOut = 0;

			cT = 's';
		}
		else if (c == 'd') // delete
		{
			if (cT == 'e')
			{
				if (fOpen)
				{
					*pOut++ = '\n';
					*pOut = 0;
				}
			
				*pOut = 0;
				fOpen = 1;
			}

			// *pOut++ = '+';
			// *pOut++ = ' ';
			pOut += strcpyTokSpace(pOut, s++);
			// *pOut++ = '\n';
			*pOut = 0;

			cT = 'd';
		}
		else if (c == 'i') // insert
		{
			if (cT == 'e')
			{
				if (fOpen)
				{
					*pOut++ = '\n';
					*pOut = 0;
				}
				
				*pOut = 0;
				fOpen = 1;
			}

			// *pOut++ = '-';
			// *pOut++ = ' ';
			pOut += strcpyTokSpace(pOut, t++);
			// *pOut++ = '\n';
			*pOut = 0;

			cT = 'i';
		}
	}

	if (fOpen)
	{
		*pOut = 0;
	}

	*pOut = 0;

	*pcTraceback1 = s - sT;
	*pcTraceback2 = t - tT;
}


void performSmartEdit(char* pOutT, char* ptraceback, TOK* s, TOK* t, int* pcTraceback1, int* pcTraceback2)
{
	++s;
	++t;
	char c = 0;
	char cT = 0;
	TOK* sT = s;
	TOK* tT = t;

	char* pOut = pOutT;

	while (c = *ptraceback++)
	{
		if (c == 'e') // emit
		{
			pOut += strcpyTok(pOut, s++);
			*pOut = 0;
			++t;

			cT = 'e';
		}
		else if (c == 's') // substitute
		{
			*pOut++ = '{';
			*pOut++ = ' ';
			*pOut = 0;
		
			if (cT != 'd')
				pOut += strcpyTokNoWhite(pOut, t++);
			else
				pOut += strcpyTokNoWhite(pOut, s++);

			*pOut++ = ' ';
			*pOut++ = '|';
			*pOut++ = ' ';
			*pOut = 0;

			if (cT != 'd')
				pOut += strcpyTokNoWhite(pOut, s++);
			else
				pOut += strcpyTokNoWhite(pOut, t++);

			*pOut++ = ' ';
			*pOut++ = '}';
			pOut += strcpyTokJustWhite(pOut, (s-1));
			*pOut = 0;

			cT = 's';
		}
		else if (c == 'd') // delete
		{
			*pOut++ = '{';
			*pOut++ = ' ';
			*pOut = 0;

			pOut += strcpyTokNoWhite(pOut, s++);

			*pOut++ = ' ';
			*pOut++ = '}';
			pOut += strcpyTokJustWhite(pOut, (s-1));
			*pOut = 0;

			cT = 'd';
		}
		else if (c == 'i') // insert
		{		
			*pOut++ = '{';
			*pOut++ = ' ';
			*pOut = 0;

			pOut += strcpyTokNoWhite(pOut, t++);

			*pOut++ = ' ';
			*pOut++ = '}';
			pOut += strcpyTokJustWhite(pOut, (t-1));
			*pOut = 0;

			cT = 'i';
		}
	}

	*pOut = 0;

	*pcTraceback1 = s - sT;
	*pcTraceback2 = t - tT;
}


int strcpyTokCloneWhite(char* p, TOK* s, TOK* t)
{
	return strcpycount(p, s->szTok, t->szWhite);
}

int strcpyJustCloneWhite(char* p, TOK* t)
{
	return strcpycount(p, "", t->szWhite);
}

// try to clone linebreaks and whitespace from File2 to File1
void performLinebreaks(char* pOutT, char* ptraceback, TOK* s, TOK* t, int* pcTraceback1, int* pcTraceback2)
{
	++s;
	++t;
	char c = 0;
	TOK* sT = s;
	TOK* tT = t;

	char* pOut = pOutT;

	while (c = *ptraceback++)
	{
		if ((t->iHash != 0xfdfdfdfd) && (t->szTok) && (t->szTok[0]) && (0 == strcmp(t->szTok, "PAGEBREAK")))
		{
			strcpy(pOut, "PAGEBREAK\n");
			pOut += 10;
		}


		if (c == 'e') // emit
		{
			pOut += strcpyTokCloneWhite(pOut, s++, t++);
		}
		else if (c == 's') // substitute
		{
			pOut += strcpyTokCloneWhite(pOut, s++, t++);
		}
		else if (c == 'd') // delete
		{
			pOut += strcpyTok(pOut, s++);
		}
		else if (c == 'i') // insert
		{
			pOut += strcpyJustCloneWhite(pOut, t++);
		}
	}

	*pcTraceback1 = s - sT;
	*pcTraceback2 = t - tT;
}

char* szUsage = "pgdiff [options] inputFile1 inputFile2 > outputFile\n"
"\n"
"Attempts to find and display in curly braces word mismatches between\n"
"inputFile1 and inputFile2 while retaining the linebreaks of inputFile1 --\n"
"IE inputFile1 and inputFile2 DO NOT have to have corresponding lines of text\n"
"simply similar lists of words within their texts. The algorithm tends to be\n"
"relatively insensitive to differences in whitespace and linebreaks.\n"
"\n"
"options:\n"
"\n"
"-w ddddd -- set the approx. size of how many words in a row that\n"
"            can be mismatched before the algorithm gives up.\n"
"            note: running time of the algorithm is proportional\n"
"            to this number -- which can be slow if greater than about 10000\n"
"\n"
"-linebreaks -- suppresses display of mismatched words in the output\n"
"            and instead simply attempts to clone the linebreaks from\n"
"            inputFile2 to inputFile1. This is useful in attempting\n"
"            to recover the linebreaks of an original text when they\n"
"            have been lost in an previously editting OCR effort.\n"
"            The outputFile with recovered linebreaks can then be\n"
"            for example resubmitted to DP.\n"
"\n"
"-smarted -- changes the output format slightly to a form more easily\n"
"            processed by editors designed for this particular output\n"
"\n"
"-justdiffs -- just output the words or regions that differ;\n"
"\n";

void ShowUsage()
{
	fprintf(stderr, "%s", szUsage);
	ErrorExit();
}

int main(int argc, char *argv[])
{
	PrepDic(argc, argv);

	int i;
	int cloop = 0;
	int maxsize = 120;

	int cThrough = 0;

	int fLinebreaks = 0;
	int fSmartEd = 0;
	int fJustDiffs = 0;

	if (argc < 3)
		ShowUsage();

	for (i = 1; i < argc-2; ++i)
	{
		if (0 == strcmp("-w", (const char*)argv[i]))
		{
			maxsize = (atoi((const char*)argv[i+1]));
			maxsize += 2;
		}
		
		if (0 == strcmp("-linebreaks", (const char*)argv[i]))
		{
			fLinebreaks = 1;
		}

		if (0 == strcmp("-smarted", (const char*)argv[i]))
		{
			fSmartEd = 1;
		}

		if (0 == strcmp("-justdiffs", (const char*)argv[i]))
		{
			fJustDiffs = 1;
		}
	}

	if (maxsize < 30)
		maxsize = 30;

	if (maxsize > 10000)
	{
		fprintf(stderr, "Setting a -w parameter larger than 10,000 results in excessive run times\n");
		ErrorExit();
	}

	int cTraceback1 = maxsize;
	int cTraceback2 = maxsize;

	assert(maxsize < 100000);

	short* d = (short*) malloc(sizeof(short) * (maxsize+1) * (maxsize+1));

	assert(d);

	char* pOut = (char*) malloc(20*(maxsize + 1));

	assert(pOut);

	TOK* s = (TOK*) malloc(sizeof(TOK) * (maxsize+1));
	TOK* t = (TOK*) malloc(sizeof(TOK) * (maxsize+1));

	assert(s);
	assert(t);

	for (i = 0; i <= maxsize; ++i)
	{
		InitTOK(s+i);
		InitTOK(t+i);
	}

	FILE* F1 = fopen((const char*)argv[argc-2], "r");
	FILE* F2 = fopen((const char*)argv[argc-1], "r");

	if (!F1)
	{
		fprintf(stderr, "Couldn't open file named '%s'\n\n", argv[argc-2]);
		ShowUsage();
		ErrorExit();
	}

	if (!F2)
	{
		fprintf(stderr, "Couldn't open file named '%s'\n\n", argv[argc-1]);
		ShowUsage();
		ErrorExit();
	}
	

	char* ptraceback = (char*) malloc(3 + 2 * maxsize);

	while ( (++cThrough >= 0) && (!( (chBuf1 == EOF) && (chBuf2 == EOF) ) ) )
	{
		++cloop;

		GetToks(&chBuf1, s+maxsize-cTraceback1, cTraceback1, F1);
		GetToks(&chBuf2, t+maxsize-cTraceback2, cTraceback2, F2);

		LD(maxsize, d, s, t);

		traceback(maxsize, ptraceback, d, maxsize, maxsize);

		int iTruncatePoint;
		if ((chBuf1 != EOF) && (chBuf2 != EOF))
			iTruncatePoint = truncateTraceback(maxsize, ptraceback);
		else
			iTruncatePoint = strlen(ptraceback);

		if (iTruncatePoint < 0)
			DumpTOKS(s, t, maxsize);

		if ((!fLinebreaks) && (!fSmartEd) && (!fJustDiffs))
			performEdit(pOut, ptraceback, s, t, &cTraceback1, &cTraceback2);
		else if ((!fSmartEd) && (!fJustDiffs))
			performLinebreaks(pOut, ptraceback, s, t, &cTraceback1, &cTraceback2);
		else if (!fJustDiffs)
			performSmartEdit(pOut, ptraceback, s, t, &cTraceback1, &cTraceback2);
		else
			performJustDiffs(pOut, ptraceback, s, t, &cTraceback1, &cTraceback2);

		printf("%s", pOut);

		ShiftToks(s, cTraceback1, maxsize);	
		ShiftToks(t, cTraceback2, maxsize);
	}

	return 0;
}