/*************************************************************************/
/* Copyright (c) 2004                                                    */
/* Daniel Sleator, David Temperley, and John Lafferty                    */
/* Copyright 2008, 2009, 2012 Linas Vepstas <linasvepstas@gmail.com>     */
/* All rights reserved                                                   */
/*                                                                       */
/* Use of the link grammar parsing system is subject to the terms of the */
/* license set forth in the LICENSE file included with this software.    */
/* This license allows free redistribution and use in source and binary  */
/* forms, with or without modification, subject to certain conditions.   */
/*                                                                       */
/*************************************************************************/

#include "api-structures.h"
#include "dict-common/dialect.h"      // dialect_alloc
#include "dict-common/dict-affix.h"
#include "dict-common/dict-affix-impl.h"
#include "dict-common/dict-api.h"
#include "dict-common/dict-common.h"
#include "dict-common/dict-defines.h"
#include "dict-common/dict-internals.h"
#include "dict-common/dict-locale.h"
#include "dict-common/dict-utils.h"
#include "dict-common/file-utils.h"
#include "dict-common/idiom.h"
#include "dict-common/regex-morph.h"
#include "dict-ram/dict-ram.h"
#include "post-process/pp_knowledge.h"
#include "read-dialect.h"
#include "read-dict.h"
#include "read-regex.h"
#include "string-set.h"
#include "tokenize/anysplit.h"        // Initialize anysplit here ...
#include "tokenize/spellcheck.h"      // Initialize spellcheck here ...

/***************************************************************
*
* Routines for manipulating Dictionary
*
****************************************************************/

/**
 * If word has a connector, return it.
 * If word has more than one connector, return NULL.
 */
static const char * word_only_connector(Dict_node * dn)
{
	Exp * e = dn->exp;
	if (CONNECTOR_type == e->type)
		return e->condesc->more->string;
	return NULL;
}

static void load_affix(Dictionary afdict, Dict_node *dn, int l)
{
	Dict_node * dnx = NULL;
	for (; NULL != dn; dn = dnx)
	{
		const char *string;
		const char *con = word_only_connector(dn);
		if (NULL == con)
		{
			/* ??? should we support here more than one class? */
			prt_error("Warning: Word \"%s\" found near line %d of %s.\n"
			          "\tWord has more than one connector.\n"
			          "\tThis word will be ignored.\n",
			          dn->string, afdict->line_number, afdict->name);
			return;
		}

		/* The affix files serve a dual purpose: they indicate both
		 * what a unit is, connector-wise, and what is strippable, as
		 * a string.  When the unit is an 'idiom' (i.e. two words,
		 * e.g. base_pair or degrees_C) then only the first word can
		 * be stripped away from a run-on expression (e.g. "86degrees C")
		 */
		if (contains_underbar(dn->string))
		{
			char *p;
			char *writeable_string = strdupa(dn->string);
			p = writeable_string+1;
			while (*p != '_' && *p != '\0') p++;
			*p = '\0';
			string = writeable_string;
		}
		else
		{
			string = dn->string;
		}

		affix_list_add(afdict, afdict_find(afdict, con,
		               /*notify_err*/true), string);

		dnx = dn->left;
		free(dn);
	}
}

/**
 * Dummy lookup function for the affix dictionary -
 * compile_regexs() needs it.
 */
static bool return_true(Dictionary dict, const char *name)
{
	return true;
}

/**
 * Process the regex file.
 * We have to compile regexs using the dictionary locale,
 * so make a temporary locale swap. XXX FIXME: Thread safety.
 */
static bool load_regexes(Dictionary dict, const char *regex_name)
{
	if (!read_regex_file(dict, regex_name)) return false;

	const char *locale = setlocale(LC_CTYPE, NULL); /* Save current locale. */
	locale = strdupa(locale); /* setlocale() uses its own memory. */
	setlocale(LC_CTYPE, dict->locale);
	lgdebug(+D_DICT, "Regexs locale \"%s\"\n", setlocale(LC_CTYPE, NULL));

	if (!compile_regexs(dict->regex_root, dict))
	{
		locale = setlocale(LC_CTYPE, locale);         /* Restore the locale. */
		assert(NULL != locale, "Cannot restore program locale");
		return false;
	}
	locale = setlocale(LC_CTYPE, locale);            /* Restore the locale. */
	assert(NULL != locale, "Cannot restore program locale");

	return true;
}

/**
 * Read dictionary entries from a utf-8 string "input".
 * All other parts are read from files.
 */
#define D_DICT 10
static Dictionary
dictionary_six_str(const char * lang,
                   const char * input,
                   const char * dict_name,
                   const char * pp_name, const char * cons_name,
                   const char * affix_name, const char * regex_name)
{
	const char * t;
	Dictionary dict;
	size_t Exp_pool_size;

	dict = (Dictionary) malloc(sizeof(struct Dictionary_s));
	memset(dict, 0, sizeof(struct Dictionary_s));

	dict->line_number = 1;

	/* Language and file-name stuff */
	dict->string_set = string_set_create();
	t = find_last_dir_separator((char *)lang);
	t = (NULL == t) ? lang : t+1;
	dict->lang = string_set_add(t, dict->string_set);
	lgdebug(D_USER_FILES, "Debug: Language: %s\n", dict->lang);
	dict->name = string_set_add(dict_name, dict->string_set);

	if (NULL != affix_name)
	{
		if (dictionary_generation_request(dict))
		{
			const size_t initial_allocation = 256;
			dict->num_categories_alloced = initial_allocation;
			dict->category = malloc(sizeof(*dict->category) * initial_allocation);
		}
		else
		{
			dict->spell_checker = spellcheck_create(dict->lang);
		}

#if defined HAVE_HUNSPELL || defined HAVE_ASPELL
		/* FIXME: Move to spellcheck-*.c */
		if (verbosity_level(D_USER_BASIC) && (NULL == dict->spell_checker))
			prt_error("Info: %s: Spell checker disabled.\n", dict->lang);
#endif
		memset(dict->current_idiom, 'A', IDIOM_LINK_SZ-1);
		dict->current_idiom[IDIOM_LINK_SZ-1] = 0;

		dict->insert_entry = insert_list;
		dict->lookup_list = dict_node_lookup;
		dict->lookup_wild = dict_node_wild_lookup;
		dict->free_lookup = dict_node_free_lookup;
		dict->exists_lookup = dict_node_exists_lookup;
		dict->clear_cache = dict_node_noop;
		dict->start_lookup = dict_lookup_noop;
		dict->end_lookup = dict_lookup_noop;

		dict->dialect_tag.set = string_id_create();

		// Actual usage:
		// Lang  Exp   Condesc
		//  en   47K    2.3K
		//  ru  300K    5.5K
		//  th    8K    2.0K
		// The pool sizes are slightly under a power of two,
		// so that malloc doesn't round up to next power of two.
		condesc_init(dict, 3060);
		Exp_pool_size = 16380;

		if (!test_enabled("no-macro-tag"))
		{
			dict->macro_tag = malloc(sizeof(*dict->macro_tag));
			memset(dict->macro_tag, 0, sizeof(*dict->macro_tag));
		}
	}
	else
	{
		/* Affix dictionary. */
		afclass_init(dict);
		dict->insert_entry = load_affix;
		dict->exists_lookup = return_true;

		// English dict is the largest of all;
		// it has 16 exprs and 8 connectors in the affix table.
		condesc_init(dict, 16);
		Exp_pool_size = 30;
	}

	dict->dfine.set = string_id_create();

	dict->Exp_pool = pool_new(__func__, "Exp", /*num_elements*/Exp_pool_size,
	                          sizeof(Exp), /*zero_out*/false,
	                          /*align*/false, /*exact*/false);

	/* Read dictionary from the input string. */
	if (!read_dictionary(dict, input))
		goto failure;

	if (NULL == affix_name)
	{
		/*
		 * The affix table is handled alone in this invocation.
		 * Skip the rest of processing!
		 * FIXME: The dictionary creating stuff needs a rearrangement.
		 */
		return dict;
	}

	if (dict->dialect_tag.num == 0)
	{
		string_id_delete(dict->dialect_tag.set);
		dict->dialect_tag.set = NULL;
	}

	if (!dictionary_setup_defines(dict))
		goto failure;

	if (!load_regexes(dict, regex_name)) goto failure;

	dict->affix_table = dictionary_six(lang, affix_name, NULL, NULL, NULL, NULL);
	if (dict->affix_table == NULL)
	{
		prt_error("Error: Could not open affix file %s\n", affix_name);
		goto failure;
	}
	if (! afdict_init(dict))
		goto failure;

	if (! anysplit_init(dict->affix_table))
		goto failure;

	dict->base_knowledge  = pp_knowledge_open(pp_name);
	dict->hpsg_knowledge  = pp_knowledge_open(cons_name);

	condesc_setup(dict);

	// Special-case hack.
	if ((0 == strncmp(dict->lang, "any", 3)) ||
	    (NULL != dict->affix_table->anysplit))
		dict->shuffle_linkages = true;

	return dict;

failure:
	dictionary_delete(dict);
	return NULL;
}

/**
 * Use filenames of six different files to put together the dictionary.
 */
NO_SAN_DICT
Dictionary
dictionary_six(const char * lang, const char * dict_name,
               const char * pp_name, const char * cons_name,
               const char * affix_name, const char * regex_name)
{
	Dictionary dict;

	char* input = get_file_contents(dict_name);
	if (NULL == input)
	{
		prt_error("Error: Could not open dictionary \"%s\"\n", dict_name);
		return NULL;
	}

	dict = dictionary_six_str(lang, input, dict_name, pp_name,
	                          cons_name, affix_name, regex_name);

	free_file_contents(input);
	return dict;
}

Dictionary dictionary_create_from_file(const char * lang)
{
	Dictionary dictionary;

	init_memusage();
	if (lang && *lang)
	{
		char * dict_name;
		char * pp_name;
		char * cons_name;
		char * affix_name;
		char * regex_name;

		dict_name = join_path(lang, "4.0.dict");
		pp_name = join_path(lang, "4.0.knowledge");
		cons_name = join_path(lang, "4.0.constituent-knowledge");
		affix_name = join_path(lang, "4.0.affix");
		regex_name = join_path(lang, "4.0.regex");

		dictionary = dictionary_six(lang, dict_name, pp_name, cons_name,
		                            affix_name, regex_name);

		free(regex_name);
		free(affix_name);
		free(cons_name);
		free(pp_name);
		free(dict_name);

		if (dictionary != NULL)
		{
			char *dialect_name = join_path(lang, "4.0.dialect");
			if (!dialect_file_read(dictionary, dialect_name))
			{
				dictionary_delete(dictionary);
				dictionary = NULL;
			}
			else if ((dictionary->dialect == NULL) ||
			         (dictionary->dialect->num_table_tags == 0))
			{
				/* No or empty dialect file. */
				free_dialect(dictionary->dialect);
				dictionary->dialect = NULL;
			}
			free(dialect_name);
		}
	}
	else
	{
		prt_error("Error: No language specified!\n");
		dictionary = NULL;
	}

	return dictionary;
}


#ifdef NOTDEF
/**
 * Use "string" as the input dictionary. All of the other parts,
 * including post-processing, affix table, etc, are NULL.
 * This routine is intended for unit-testing ONLY.
 *
 * FIXME? (Not used for now.)
 * 1. get_default_locale() returns locale, not language.
 * 2. "lang" memory-leak on success.
 */
Dictionary dictionary_create_from_utf8(const char * input)
{
	Dictionary dictionary = NULL;
	char * lang;

	init_memusage();

	lang = get_default_locale();
	if (lang && *lang) {
		dictionary = dictionary_six_str(lang, input, "string",
		                                NULL, NULL, NULL, NULL);
		free(lang);
	} else {
		/* Default to en when locales are broken (e.g. WIN32) */
		dictionary = dictionary_six_str("en", input, "string",
		                                NULL, NULL, NULL, NULL);
	}

	return dictionary;
}
#endif // NOTDEF

