// Copyright 2018 David Sietz and [`test-data-generator` contributors](https://github.com/dsietz/test-data-generator/blob/master/CONTRIBUTORS.md).
// Licensed under the MIT license
// (see LICENSE or <https://opensource.org/licenses/Apache-2.0>)
//
//!
//! The are multiple ways to use the Test Data Generation library. It all depends on your intent.
//!
//! ### Profile
//!
//! The easiest way is to use a Profile. The `profile` module provides functionality to create a profile on a data sample (Strings).
//! Once a profile has been made, data can be generated by calling the _pre_generate()_ and _generate()_ functions, in that order.
//!
//! ```
//! extern crate test_data_generation;
//!
//! use test_data_generation::Profile;
//!
//! fn main() {
//!     // analyze the dataset
//! 	let mut data_profile =  Profile::new();
//!
//!     // analyze the dataset
//! 	data_profile.analyze("Smith, John");
//! 	data_profile.analyze("Doe, John");
//! 	data_profile.analyze("Dale, Danny");
//! 	data_profile.analyze("Rickets, Ronney");
//!
//!     // confirm 4 data samples were analyzed
//!    	assert_eq!(data_profile.patterns.len(), 4);
//!
//!     // prepare the generator
//!     data_profile.pre_generate();
//!
//!     // generate some data
//!    	println!("The generated name is {:?}", data_profile.generate());
//! }
//! ```
//!
//! You can also export (archive as JSON file) the profile for later use.
//! This allows for the algorithm to be retrieved without having to store the actual data that was analyzed.
//!
//!	```
//! extern crate test_data_generation;
//!
//! use test_data_generation::Profile;
//!
//! fn main() {
//!		//create a profile and analyze some data
//!		let mut old_profile =  Profile::new();
//!		old_profile.analyze("Smith, John");
//!		old_profile.analyze("O'Brian, Henny");
//!		old_profile.analyze("Dale, Danny");
//!		old_profile.analyze("Rickets, Ronney");
//!
//!		old_profile.pre_generate();
//!
//!		//save the profile for later
//!		assert_eq!(old_profile.save("./tests/samples/sample-00-profile").unwrap(), true);
//!
//!		// create a new profile from the archive json file
//!		let mut new_profile = Profile::from_file("./tests/samples/sample-00-profile");
//!
//!		// generate some data. NOTE that the pre-generate() was already called prior to saving
//!     println!("The generated name is {:?}", new_profile.generate());
//! }
//! ```
//!
//! ### Data Sample Parser
//!
//! If you are using CSV files of data samples, then you may wish to use a Data Sample Parser.
//! The `data_sample_parser` module provides functionality to read sample data, parse and analyze it, so that test data can be generated based on profiles.
//!
//! ```
//! extern crate test_data_generation;
//! use test_data_generation::data_sample_parser::DataSampleParser;
//!
//! fn main() {
//!     let mut dsp = DataSampleParser::new();
//!     dsp.analyze_csv_file(&String::from("./tests/samples/sample-01.csv")).unwrap();
//!
//!     println!("My new name is {} {}", dsp.generate_record()[0], dsp.generate_record()[1]);
//!     // My new name is Abbon Aady
//! }
//! ```
//!
//! You can also save the Data Sample Parser (the algorithm) as an archive file (json) ...
//!
//! ```
//! extern crate test_data_generation;
//! use test_data_generation::data_sample_parser::DataSampleParser;
//!
//! fn main() {
//!     let mut dsp =  DataSampleParser::new();
//!     dsp.analyze_csv_file(&String::from("./tests/samples/sample-01.csv")).unwrap();
//!
//!     assert_eq!(dsp.save(&String::from("./tests/samples/sample-01-dsp")).unwrap(), true);
//! }
//! ```
//!
//! and use it at a later time.
//!
//! ```
//! extern crate test_data_generation;
//! use test_data_generation::data_sample_parser::DataSampleParser;
//!
//! fn main() {
//!     let mut dsp = DataSampleParser::from_file(&String::from("./tests/samples/sample-01-dsp"));
//!
//! 	println!("Sample data is {:?}", dsp.generate_record()[0]);
//! }
//! ```
//!
//! You can also generate a new csv file based on the data sample provided.
//!
//! ```
//! extern crate test_data_generation;
//!
//! use test_data_generation::data_sample_parser::DataSampleParser;
//!
//! fn main() {
//!     let mut dsp =  DataSampleParser::new();
//!
//!    	dsp.analyze_csv_file(&String::from("./tests/samples/sample-01.csv")).unwrap();
//!    	dsp.generate_csv(100, &String::from("./tests/samples/generated-01.csv")).unwrap();
//! }
//! ```
#![crate_type= "lib"]
#![crate_name = "test_data_generation"]

#[macro_use]
extern crate log;

#[macro_use]
extern crate serde_derive;
extern crate serde;
extern crate serde_json;
extern crate serde_yaml;
extern crate yaml_rust;
extern crate regex;
extern crate rand;
extern crate crossbeam;
extern crate csv;
extern crate levenshtein;

use crate::engine::{Fact, PatternDefinition};
use std::collections::BTreeMap;
use std::ops::AddAssign;
use std::fs::File;
use std::io;
use std::io::Write;
use std::io::prelude::*;

type PatternMap = BTreeMap<String, u32>;
type SizeMap = BTreeMap<u32, u32>;
type SizeRankMap  = BTreeMap<u32, f64>;

#[derive(Clone, Serialize, Deserialize, Debug)]
/// Represents a Profile for sample data that has been analyzed and can be used to generate realistic data
pub struct Profile {	
	/// An identifier (not necessarily unique) that is used to differentiate profiles from one another
	pub id: Option<String>,
    /// A list of symbolic patterns with a distinct count of occurrences
	pub patterns: PatternMap,
	/// The total number of patterns in the profile
	pub pattern_total: u32,
	/// A list of symbolic patterns in the profile
	/// (used for temporary storage due to lifetime issues)
	pub pattern_keys: Vec<String>,
	/// A list of distinct counts for patterns in the profile
	/// (used for temporary storage due to lifetime issues)
	pub pattern_vals: Vec<u32>,
	/// A list of symbolic patterns with their percent chance of occurrence
	pub pattern_percentages: Vec<(String, f64)>,
	/// A list of symbolic patterns with a running total of percent chance of occurrence, in increasing order
	pub pattern_ranks: Vec<(String, f64)>,
	/// A list of pattern lengths with a distinct count of occurrence
	pub sizes: SizeMap,
	/// the total number of pattern sizes (lengths) in the profile
	pub size_total: u32,
	/// A list of pattern sizes (lengths) with a running total of their percent chance of occurrence, in increasing order
	pub size_ranks: Vec<(u32, f64)>,
	/// The number of processors used to distribute the work load (multi-thread) while finding Facts to generate data
	pub processors: u8,
	/// A list of processors (which are lists of Facts) that store all the Facts in the profile
	pub facts: Vec<Vec<Fact>>,
}

impl Profile {
	/// Constructs a new Profile
	///
	/// #Example
	///
	/// ```rust
	/// extern crate test_data_generation;
	///
	/// use test_data_generation::Profile;
	///
	/// fn main() {
	/// 	let placeholder = Profile::new();
	/// }
	/// ```
	pub fn new() -> Profile {
		Profile {
			id: None,
			patterns: PatternMap::new(),
			pattern_total: 0,
			pattern_keys: Vec::new(),
			pattern_vals: Vec::new(),
			pattern_percentages: Vec::new(),
			pattern_ranks: Vec::new(),
			sizes: SizeMap::new(),
			size_total: 0,
			size_ranks: Vec::new(),
			processors: 4,
			facts: Profile::new_facts(4),
		}
	}

	/// Constructs a new Profile using an identifier
	///
	/// #Example
	///
	/// ```rust
	/// extern crate test_data_generation;
	///
	/// use test_data_generation::Profile;
	///
	/// fn main() {
	/// 	let placeholder = Profile::new_with_id("12345".to_string());
	/// }
	/// ```
	pub fn new_with_id(id: String) -> Profile {
		Profile {
			id: Some(id),
			patterns: PatternMap::new(),
			pattern_total: 0,
			pattern_keys: Vec::new(),
			pattern_vals: Vec::new(),
			pattern_percentages: Vec::new(),
			pattern_ranks: Vec::new(),
			sizes: SizeMap::new(),
			size_total: 0,
			size_ranks: Vec::new(),
			processors: 4,
			facts: Profile::new_facts(4),
		}
	}

	/// Constructs a new Profile with a specified number of processors to analyze the data.
	/// Each processor shares the load of generating the data based on the Facts it has been assigned to manage.
	///
	/// # Arguments
	///
	/// * `p: u8` - A number that sets the number of processors to start up to manage the Facts.</br>
	///         Increasing the number of processors will speed up the generator be distributing the workload.
	///         The recommended number of processors is 1 per 10K data points (e.g.: profiling 20K names should be handled by 2 processors)</br>
	///         NOTE: The default number of processors is 4.
	///
	/// #Example
	///
	/// ```rust
	/// extern crate test_data_generation;
	///
	/// use test_data_generation::Profile;
	///
	/// fn main() {
	///     let processors: u8 = 10;
	/// 	let placeholder = Profile::new_with_processors(processors);
	/// }
	/// ```
	pub fn new_with_processors(p: u8) -> Profile {
		Profile {
			id: None,
			patterns: PatternMap::new(),
			pattern_total: 0,
			pattern_keys: Vec::new(),
			pattern_vals: Vec::new(),
			pattern_percentages: Vec::new(),
			pattern_ranks: Vec::new(),
			sizes: SizeMap::new(),
			size_total: 0,
			size_ranks: Vec::new(),
			processors: p,
			facts: Profile::new_facts(p),
		}
	}

	/// Constructs a new Profile from an exported JSON file. This is used when restoring from "archive"
	///
	/// # Arguments
	///
	/// * `field: String` - The full path of the export file , excluding the file extension, (e.g.: "./test/data/custom-names").</br>
	///
	/// #Example
	///
	/// ```rust
	/// extern crate test_data_generation;
	///
	/// use test_data_generation::Profile;
	///
	/// fn main() {
	///		let mut profile = Profile::from_file("./tests/samples/sample-00-profile");
    ///
    ///     profile.pre_generate();
    ///
    ///     println!("The generated name is {:?}", profile.generate());
	/// }
    /// ```
	pub fn from_file(path: &'static str) -> Profile {
		// open the archive file
		let mut file = match File::open(format!("{}.json",&path)) {
			Err(_e) => {
				error!("Could not open file {:?}", &path.to_string());
				panic!("Could not open file {:?}", &path.to_string());
			},
			Ok(f) => {
				info!("Successfully opened file {:?}", &path.to_string());
				f
			},
		};

		//read the archive file
		let mut serialized = String::new();
		match file.read_to_string(&mut serialized) {
			Err(e) => {
				error!("Could not read file {:?} because of {:?}", &path.to_string(), e.to_string());
				panic!("Could not read file {:?} because of {:?}", &path.to_string(), e.to_string());
			},
			Ok(s) => {
				info!("Successfully read file {:?}", &path.to_string());
				s
			},
		};

        //serde_json::from_str(&serialized).unwrap()
        Self::from_serialized(&serialized)
	}


	/// Constructs a new Profile from a serialized (JSON) string of the Profile object. This is used when restoring from "archive"
	///
	/// #Example
	///
	/// ```rust
	/// extern crate test_data_generation;
	///
	/// use test_data_generation::Profile;
	///
	/// fn main() {
	///		let serialized = "{\"patterns\":{\"VC\":1},\"pattern_total\":1,\"pattern_keys\":[\"VC\"],\"pattern_vals\":[1],\"pattern_percentages\":[],\"pattern_ranks\":[],\"sizes\":{\"2\":1},\"size_total\":1,\"size_ranks\":[],\"processors\":4,\"facts\":[[{\"key\":\"O\",\"prior_key\":null,\"next_key\":\"K\",\"pattern_placeholder\":\"V\",\"starts_with\":1,\"ends_with\":0,\"index_offset\":0}],[{\"key\":\"K\",\"prior_key\":\"O\",\"next_key\":null,\"pattern_placeholder\":\"C\",\"starts_with\":0,\"ends_with\":1,\"index_offset\":1}],[],[]]}";
    ///		let mut profile = Profile::from_serialized(&serialized);
    ///
    ///     profile.pre_generate();
    ///
    ///     println!("The generated name is {:?}", profile.generate());
	/// }
    /// ```
	pub fn from_serialized(serialized: &str) -> Profile {
		serde_json::from_str(&serialized).unwrap()
	}

	/// This function converts an data point (&str) to a pattern and adds it to the profile
	///
	/// # Arguments
	///
	/// * `entity: String` - The textual str of the value to anaylze.</br>
	///
	/// # Example
	///
	/// ```rust
	/// extern crate test_data_generation;
	///
	/// use test_data_generation::Profile;
	///
	/// fn main() {
    /// 	let mut profile =  Profile::new();
    ///		profile.analyze("One");
    ///		profile.analyze("Two");
    ///		profile.analyze("Three");
    ///		profile.analyze("Four");
    ///
    ///		assert_eq!(profile.patterns.len(), 4);
	/// }
	/// ```
	pub fn analyze(&mut self, entity: &str) {
		let rslt = PatternDefinition::new().analyze(entity);
		let _t = self.apply_facts(rslt.0, rslt.1).map_err(|e| {
			error!("Warning: Couldn't apply the pattern and facts for the entity {}!", entity);
    		e.to_string()
		});
	}

	/// This function applies the pattern and list of Facts  to the profile
	///
	/// # Arguments
	///
	/// * `pattern: String` - The string the represents the pattern of the entity that was analyzed.</br>
	/// * `facts: Vec<Fact>` - A Vector containing the Facts based on the analysis (one for each char in the entity).</br>
	///
	/// # Example
	///
	/// ```rust
	/// extern crate test_data_generation;
	///
    /// use test_data_generation::engine::{Fact, PatternDefinition};
	/// use test_data_generation::Profile;
	///
	/// fn main() {
	/// 	let mut profile =  Profile::new();
	///		let results = PatternDefinition::new().analyze("Word");
	///
	///		assert_eq!(profile.apply_facts(results.0, results.1).unwrap(), 1);
	/// }
	/// ```
	pub fn apply_facts(&mut self, pattern: String, facts: Vec<Fact>) -> Result<i32, String>{
		// balance the storing of facts across all the vectors that can be processed in parallel
		let mut i = 0;
		for f in facts.into_iter() {
			if i == self.processors {
				i = 0;
			}

			self.facts[i as usize].push(f);
			i = i + 1;
		}

		// store the pattern
		AddAssign::add_assign(self.patterns.entry(pattern.to_string()).or_insert(0), 1);

		// store the total number of patterns generated so far
		self.pattern_total = self.patterns.values().sum::<u32>();

		// analyze sizes
		AddAssign::add_assign(self.sizes.entry(pattern.len() as u32).or_insert(0), 1);
		self.size_total = self.sizes.values().sum::<u32>();

		self.pattern_keys = self.patterns.keys().cloned().collect();
		self.pattern_vals = self.patterns.values().cloned().collect();

		Ok(1)
	}

	/// This function calculates the patterns to use by the chance they will occur (as cumulative percentage) in decreasing order
	///
	/// # Example
	///
	/// ```rust
	/// extern crate test_data_generation;
	///
	/// use test_data_generation::Profile;
	///
	/// fn main() {
    /// 	let mut profile =  Profile::new();
    ///
    ///    	profile.analyze("Smith, John");
    ///    	profile.analyze("O'Brian, Henny");
    ///    	profile.analyze("Dale, Danny");
    ///    	profile.analyze("Rickets, Ronnae");
    ///    	profile.analyze("Richard, Richie");
    ///    	profile.analyze("Roberts, Blake");
    ///    	profile.analyze("Conways, Sephen");
    ///
    ///    	profile.pre_generate();
    ///    	let test = [("CvccvccpSCvccvv".to_string(), 28.57142857142857 as f64), ("CcvccpSCvcc".to_string(), 42.857142857142854 as f64), ("CvccvccpSCvccvc".to_string(), 57.14285714285714 as f64), ("CvcvcccpSCcvcv".to_string(), 71.42857142857142 as f64), ("CvcvpSCvccc".to_string(), 85.7142857142857 as f64), ("V@CcvvcpSCvccc".to_string(), 99.99999999999997 as f64)];
    ///
    ///    	assert_eq!(profile.pattern_ranks, test);
    /// }
	/// ```
	pub fn cum_patternmap(&mut self) {
		// Reference: https://users.rust-lang.org/t/cannot-infer-an-appropriate-lifetime-for-autoref/13360/3

		debug!("calucating the cumulative percentage of occurences for data point patterns...");

		// calculate the percentage by patterns
		// -> {"CcvccpSCvcc": 14.285714285714285, "CvccvccpSCvccvc": 14.285714285714285, "CvccvccpSCvccvv": 28.57142857142857, "CvcvcccpSCcvcv": 14.285714285714285, "CvcvpSCvccc": 14.285714285714285, "V~CcvvcpSCvccc": 14.285714285714285}
		let n = self.patterns.len();

		// see issue: https://github.com/dsietz/test-data-generation/issues/88
		self.pattern_percentages.clear();

		for m in 0..n {
			self.pattern_percentages.push((self.pattern_keys[m].clone(), (self.pattern_vals[m] as f64 / self.pattern_total as f64) * 100.0));
		}

		// sort the ranks by percentages in decreasing order
		// -> [("CvccvccpSCvccvv", 28.57142857142857), ("CcvccpSCvcc", 14.285714285714285), ("CvccvccpSCvccvc", 14.285714285714285), ("CvcvcccpSCcvcv", 14.285714285714285), ("CvcvpSCvccc", 14.285714285714285), ("V~CcvvcpSCvccc", 14.285714285714285)]
		self.pattern_percentages.sort_by(|&(_, a), &(_, b)| b.partial_cmp(&a).unwrap());

		// calculate the cumulative sum of the pattern rankings
		// -> [("CvccvccpSCvccvv", 28.57142857142857), ("CcvccpSCvcc", 42.857142857142854), ("CvccvccpSCvccvc", 57.14285714285714), ("CvcvcccpSCcvcv", 71.42857142857142), ("CvcvpSCvccc", 85.7142857142857), ("V~CcvvcpSCvccc", 99.99999999999997)]
		let mut rank: f64 = 0.00;

		// see issue: https://github.com/dsietz/test-data-generation/issues/88
		self.pattern_ranks.clear();

		for pttrn in self.pattern_percentages.iter() {
			let tmp = pttrn.1 + rank;
			self.pattern_ranks.push((pttrn.0.clone(),tmp));
			rank = tmp;
		}
	}

    /// This function calculates the sizes to use by the chance they will occur (as cumulative percentage) in decreasing order
	///
	/// # Example
	///
	/// ```rust
	/// extern crate test_data_generation;
	///
	/// use test_data_generation::Profile;
	///
	/// fn main() {
    /// 	let mut profile =  Profile::new();
    ///		profile.analyze("One");
    ///		profile.analyze("Two");
    ///		profile.analyze("Three");
    ///		profile.analyze("Four");
    ///		profile.analyze("Five");
    ///		profile.analyze("Six");
    ///
    ///     profile.cum_sizemap();
    ///
    ///		print!("The size ranks are {:?}", profile.size_ranks);
    ///     // The size ranks are [(3, 50), (4, 83.33333333333333), (5, 100)]
    /// }
	/// ```
	pub fn cum_sizemap(&mut self) {
		debug!("calucating the cumulative percentage of occurences for data point sizes...");
		// calculate the percentage by sizes
		// -> {11: 28.57142857142857, 14: 14.285714285714285, 15: 57.14285714285714}
		let mut size_ranks = SizeRankMap::new();

		for key in self.sizes.keys(){
			size_ranks.insert(*key, (*self.sizes.get(key).unwrap() as f64 / self.size_total as f64)*100.0);
		}

		// sort the ranks by percentages in decreasing order
		// -> [(15, 57.14285714285714), (11, 28.57142857142857), (14, 14.285714285714285)]
		let mut sizes = size_ranks.iter().collect::<Vec<_>>();
		sizes.sort_by(|&(_, a), &(_, b)| b.partial_cmp(&a).unwrap());

		// calculate the cumulative sum of the size rankings
		// -> [(15, 57.14285714285714), (11, 85.71428571428571), (14, 100)]
		self.size_ranks = sizes.iter().scan((0 as u32, 0.00 as f64), |state, &(&k, &v)| {
			*state = (k, state.1 + &v);
			Some(*state)
		}).collect::<Vec<(_,_)>>();
	}

	/// This function generates realistic test data based on the sampel data that was analyzed.
	///
	/// # Example
	///
	/// ```rust
	/// extern crate test_data_generation;
	///
	/// use test_data_generation::Profile;
	///
	/// fn main() {
    /// 	let mut profile =  Profile::new();
    ///
    ///		profile.analyze("One");
    ///		profile.analyze("Two");
    ///		profile.analyze("Three");
    ///		profile.analyze("Four");
    ///		profile.analyze("Five");
    ///
    ///     profile.pre_generate();
    ///
    ///		print!("The test data {:?} was generated.", profile.generate());
    /// }
	/// ```
	pub fn generate(&mut self) -> String{
		// 1. get a random number
	 	let s: f64 = random_percentage!();

	 	// 2. find the first pattern that falls within the percentage chance of occurring
	 	// NOTE: The following 2 lines has been commented out because this doesn't need to
	 	//       happen since the patterns are already ranks by percent chance of occurring
	 	//       and therefore sizes (lengths) as well since the patterns include the full
	 	//       length of the entitiy analyzed.
		//let size = self.size_ranks.iter().find(|&&x|&x.1 >= &s).unwrap().0;
		//let pattern = self.pattern_ranks.iter().find(|x|&x.1 >= &s && x.0.len() == size as usize).unwrap().clone();
		let pattern = self.pattern_ranks.iter().find(|x|&x.1 >= &s).unwrap().clone();

		// lastly, generate the test data using facts that adhere to the pattern
		let generated = self.generate_from_pattern(pattern.0);

		generated
	}

	/// This function generates realistic test data based on the sample data that was analyzed.
	///
	/// # Arguments
	///
	/// * `pattern: String` - The pattern to reference when generating the test data.</br>
	///
	/// # Example
	///
	/// ```rust
	/// extern crate test_data_generation;
	///
	/// use test_data_generation::Profile;
	///
	/// fn main() {
    /// 	let mut profile =  Profile::new();
    ///
    ///		profile.analyze("01/13/2017");
    ///		profile.analyze("11/24/2017");
    ///		profile.analyze("08/05/2017");
    ///
    ///     profile.pre_generate();
    ///
    ///  	let generated = profile.generate_from_pattern("##p##p####".to_string());
    ///
    ///     assert_eq!(generated.len(), 10);
    /// }
	/// ```
	pub fn generate_from_pattern(&self, pattern: String) -> String {
		let pattern_chars = pattern.chars().collect::<Vec<char>>();
		let mut generated = String::new();
		let prev_char = ' ';

		// iterate through the chars in the pattern string
		for (idx, ch) in pattern_chars.iter().enumerate() {
			match crossbeam::scope(|scope| {
				let c = ch;
				let starts = if idx == 0 { 1 } else { 0 };
			 	let ends = if idx == pattern_chars.len()-1 { 1 } else { 0 };
			 	let mut fact_options = vec![];
			 	let prior_char = prev_char;

			 	// iterate through the processors (vec) that hold the lists (vec) of facts
				for v in &self.facts {
					let selected_facts = scope.spawn(move |_| {
						let mut facts = vec![];

						// iterate through the list of facts
						for value in v {
							if value.starts_with == starts &&
							   value.ends_with == ends &&
							   value.pattern_placeholder == *c &&
							   value.index_offset == idx as u32 {
									facts.push(value.key.clone());

									// if the value.key's prior char matches the prior generated char, then weight the value.key
									// to increase the chance of it being used when generated
									if value.prior_key.unwrap_or(' ') == prior_char {
										facts.push(value.key.clone());
										facts.push(value.key.clone());
									}

									// if the value.key's index_offset matches the current index, then weight the value.key
									// to increase the chance of it being used when generated
									if value.index_offset == idx as u32 {
										facts.push(value.key.clone());
										facts.push(value.key.clone());
									}
							}
						}

						facts
					});

					//append the selected_facts to the fact_options 
					//fact_options.extend_from_slice(&selected_facts.join());
					match selected_facts.join() {
						Ok(sf) => fact_options.extend_from_slice(&sf),
						Err(err) => {
							error!("{:?}", err);
							panic!("{:?}", err);
						}
					}
				}

				//select a fact to use as the generated char
				let rnd_start = 0;
				let rnd_end = fact_options.len()-1;

				if rnd_start >= rnd_end {
					//generated.push(fact_options[0 as usize]);					
					fact_options[0 as usize]
				}else{
					let x: u32 = random_between!(rnd_start, rnd_end);
					//prev_char = fact_options[x as usize];
					//generated.push(prev_char);
					fact_options[x as usize]
				}
			}) {
				Ok(c) => generated.push(c),
				Err(err) => {
					error!("{:?}", err);
					panic!("{:?}", err);
				},
			}
		}

		generated
	}

	/// This function learns by measuring how realistic the test data it generates to the sample data that was provided.
	///
	/// # Arguments
	///
	/// * `control_list: Vec<String>` - The list of strings to compare against. This would be the real data from the data sample.</br>
	///
	/// # Example
	///
	/// ```rust
	/// extern crate test_data_generation;
	///
	/// use test_data_generation::Profile;
	///
	/// fn main() {
	/// 	let mut profil =  Profile::new();
	/// 	let sample_data = vec!("Smith, John".to_string(),"Doe, John".to_string(),"Dale, Danny".to_string(),"Rickets, Ronney".to_string());
	///
	/// 	for sample in sample_data.iter().clone() {
	/// 		profil.analyze(&sample);
	/// 	}
	///
	/// 	// in order to learn the profile must be prepared with pre_genrate()
	///		// so it can generate data to learn from
	///		profil.pre_generate();
	///
	/// 	let learning = profil.learn_from_entity(sample_data).unwrap();
	///
	/// 	assert_eq!(learning, true);
    /// }
	/// ```
	pub fn learn_from_entity(&mut self, control_list: Vec<String>) -> Result<bool, String> {
		for _n in 0..10 {
			let experiment = self.generate();
			let mut percent_similarity: Vec<f64> = Vec::new();

			for control in control_list.iter().clone() {
				debug!("Comparing {} with {} ...", &control, &experiment);
				percent_similarity.push(self.realistic_test(&control, &experiment));
			}

			let percent = percent_similarity.iter().sum::<f64>() as f64 / percent_similarity.len() as f64;
			debug!("Percent similarity is {} ...", &percent);

			if percent >= 80 as f64 {
				self.analyze(&experiment);
			}
		}

		Ok(true)
	}

	/// This function calculates the levenshtein distance between 2 strings.
	/// See: https://crates.io/crates/levenshtein
	///
	/// # Arguments
	///
	/// * `control: &String` - The string to compare against. This would be the real data from the data sample.</br>
	/// * `experiment: &String` - The string to compare. This would be the generated data for which you want to find the distance.</br>
	///
	/// #Example
	///
	/// ```rust
	/// extern crate test_data_generation;
	///
	/// use test_data_generation::Profile;
	///
	/// fn main() {
	///		let mut profile =  Profile::new();
	///
	///     assert_eq!(profile.levenshtein_distance(&"kitten".to_string(), &"sitting".to_string()), 3 as usize);
	/// }
	///
	pub fn levenshtein_distance(&mut self, control: &String, experiment: &String) -> usize {
		// https://docs.rs/levenshtein/1.0.3/levenshtein/fn.levenshtein.html
		levenshtein_distance!(control, experiment)
	}

	/// This function calculates the percent difference between 2 strings.
	///
	/// # Arguments
	///
	/// * `control: &String` - The string to compare against. This would be the real data from the data sample.</br>
	/// * `experiment: &String` - The string to compare. This would be the generated data for which you want to find the percent difference.</br>
	///
	/// #Example
	///
	/// ```rust
	/// extern crate test_data_generation;
	///
	/// use test_data_generation::Profile;
	///
	/// fn main() {
	///		let mut profile =  Profile::new();
	///
	///     assert_eq!(profile.realistic_test(&"kitten".to_string(), &"sitting".to_string()), 76.92307692307692 as f64);
	/// }
	///
	pub fn realistic_test(&mut self, control: &String, experiment: &String) -> f64 {
		realistic_test!(control, experiment)
	}

	/// This function is called from within the implementated structure and returns a list processors (Vec) with empty lists (Vec) for their Facts.
	/// Each processor shares the load of generating the data based on the Facts it has been assigned to manage.
	///
	/// # Arguments
	///
	/// * `p: u8` - A number that sets the number of processors to start up to manage the Facts.</br>
	///         Increasing the number of processors will speed up the generator be ditributing the workload.
	///         The recommended number of processors is 1 per 10K data points (e.g.: profiling 20K names should be handled by 2 processors)</br>
	///         NOTE: The default number of processors is 4.
	///
	fn new_facts(p: u8) -> Vec<Vec<Fact>> {
		let mut vec_main = Vec::new();

		for _ in 0..p {
			vec_main.push(Vec::new());
		}

		vec_main
	}

	/// This function prepares the size a pattern accumulated percentages order by percentage increasing
	///
	/// # Example
	///
	/// ```rust
	/// extern crate test_data_generation;
	///
	/// use test_data_generation::Profile;
	///
	/// fn main() {
    /// 	let mut profile =  Profile::new();
    ///		profile.analyze("One");
    ///		profile.analyze("Two");
    ///		profile.analyze("Three");
    ///		profile.analyze("Four");
    ///		profile.analyze("Five");
    ///		profile.analyze("Six");
    ///
    ///     profile.pre_generate();
    ///
    ///		print!("The size ranks are {:?}", profile.size_ranks);
    ///     // The size ranks are [(3, 50), (4, 83.33333333333333), (5, 100)]
    /// }
	/// ```
	pub fn pre_generate(&mut self){
		info!("Preparing the profile for data generation...");
		self.cum_sizemap();
		self.cum_patternmap();
		info!("Profile: preparing generator...");
	}

	/// This function resets the patterns that the Profile has analyzed.
	/// Call this method whenever you wish to "clear" the Profile
	///
	/// # Example
	///
	/// ```rust
	/// extern crate test_data_generation;
	///
	/// use test_data_generation::Profile;
	///
	/// fn main() {
    /// 	let mut profile =  Profile::new();
    ///
    ///		profile.analyze("One");
    ///		profile.analyze("Two");
    ///		profile.analyze("Three");
    ///
    ///     let x = profile.patterns.len();
    ///
    ///     profile.reset_analyze();
    ///
    ///		profile.analyze("Four");
    ///		profile.analyze("Five");
    ///		profile.analyze("Six");
    ///		profile.analyze("Seven");
    ///		profile.analyze("Eight");
    ///		profile.analyze("Nine");
    ///		profile.analyze("Ten");
    ///
    ///     let y = profile.patterns.len();
    ///
    ///     assert_eq!(x, 3);
    ///     assert_eq!(y, 5);
    /// }
	/// ```
	pub fn reset_analyze(&mut self) {
		info!("Resetting the profile ...");
		self.patterns = PatternMap::new();
		info!("Profile: patterns have been reset ...");
	}

	/// This function saves (exports) the Profile to a JSON file.
	/// This is useful when you wish to reuse the algorithm to generate more test data later.
	///
	/// # Arguments
	///
	/// * `field: String` - The full path of the export file , excluding the file extension, (e.g.: "./test/data/custom-names").</br>
	///
	/// #Errors
	/// If this function encounters any form of I/O or other error, an error variant will be returned.
	/// Otherwise, the function returns Ok(true).</br>
	///
	/// #Example
	///
	/// ```rust
	/// extern crate test_data_generation;
	///
	/// use test_data_generation::Profile;
	///
	/// fn main() {
	/// 	// analyze the dataset
	///		let mut profile =  Profile::new();
	///     profile.analyze("Smith, John");
    ///		profile.analyze("O'Brian, Henny");
    ///		profile.analyze("Dale, Danny");
    ///		profile.analyze("Rickets, Ronney");
    ///
    ///		profile.pre_generate();
	///
    ///     assert_eq!(profile.save("./tests/samples/sample-00-profile").unwrap(), true);
	/// }
	///
	pub fn save(&mut self, path: &'static str) -> Result<bool, io::Error>  {
		let dsp_json = serde_json::to_string(&self).unwrap();

		// Create the archive file
		let mut file = match File::create(format!("{}.json",&path)) {
			Err(e) => {
				error!("Could not create file {:?}", &path.to_string());
				return Err(e);
			},
			Ok(f) => {
				info!("Successfully exported to {:?}", &path.to_string());
				f
			},
		};

		// Write the json string to file, returns io::Result<()>
    	match file.write_all(dsp_json.as_bytes()) {
        	Err(e) => {
            	error!("Could not write to file {}", &path.to_string());
            	return Err(e);
        	},
        	Ok(_) => {
        		info!("Successfully exported to {}", &path.to_string());
        	},
    	};

		Ok(true)
	}

	/// This function converts the Profile to a serialize JSON string.
	///
	/// #Example
	///
	/// ```rust
	/// extern crate test_data_generation;
	///
	/// use test_data_generation::Profile;
	///
	/// fn main() {
	/// 	// analyze the dataset
	///		let mut data_profile =  Profile::new();
	///
	///     // analyze the dataset
	///		data_profile.analyze("OK");
	///
    ///     println!("{}", data_profile.serialize());
    ///     // {"patterns":{"VC":1},"pattern_total":1,"pattern_keys":["VC"],"pattern_vals":[1],"pattern_percentages":[],"pattern_ranks":[],"sizes":{"2":1},"size_total":1,"size_ranks":[],"processors":4,"facts":[[{"key":"O","prior_key":null,"next_key":"K","pattern_placeholder":"V","starts_with":1,"ends_with":0,"index_offset":0}],[{"key":"K","prior_key":"O","next_key":null,"pattern_placeholder":"C","starts_with":0,"ends_with":1,"index_offset":1}],[],[]]}
	/// }
	///
	pub fn serialize(&mut self) ->String {
		serde_json::to_string(&self).unwrap()
	}
}

#[macro_use]
pub mod macros;
pub mod shared;
pub mod data_sample_parser;
pub mod configs;
pub mod engine;

// Unit Tests
#[cfg(test)]
mod tests {
    use super::*;

	#[test]
	fn apply_facts(){
		let mut profile =  Profile::new();
		let results = PatternDefinition::new().analyze("Word");

		assert_eq!(profile.apply_facts(results.0, results.1).unwrap(),1);
	}

	#[test]
    fn levenshtein_test(){
    	let mut profil =  Profile::new();

    	assert_eq!(profil.levenshtein_distance(&"kitten".to_string(), &"sitting".to_string()), 3 as usize);
    }

	#[test]
	fn realistic_data_test(){
		let mut profil =  Profile::new();

		assert_eq!(profil.realistic_test(&"kitten".to_string(), &"sitting".to_string()), 76.92307692307692 as f64);
	}

	#[test]
	fn learn_from_entity(){
		let mut profil =  Profile::new();
		let sample_data = vec!("Smith, John".to_string(),"Doe, John".to_string(),"Dale, Danny".to_string(),"Rickets, Ronney".to_string());

		for sample in sample_data.iter().clone() {
			profil.analyze(&sample);
		}

		profil.pre_generate();

		let learning = profil.learn_from_entity(sample_data).unwrap();

		assert_eq!(learning, true);
	}

	#[test]
    fn logging_test(){
    	let mut profile =  Profile::new();
    	profile.reset_analyze();

    	assert!(true);
	}
	
	#[test]
    fn new_profile_with_id(){
		let mut profile = Profile::new_with_id("12345".to_string());
    	profile.pre_generate();

    	assert_eq!(profile.id.unwrap(), "12345".to_string());
    }

	#[test]
    fn new_profile_from_file(){
		let mut profile = Profile::from_file("./tests/samples/sample-00-profile");
    	profile.pre_generate();

    	assert!(profile.generate().len() > 0);
    }

    #[test]
    #[should_panic]
    fn new_profile_from_file_bad_data(){
		let mut profile = Profile::from_file("./tests/samples/not-readable");
    	profile.pre_generate();

    	assert!(profile.generate().len() > 0);
    }

    #[test]
    #[should_panic(expected = "Could not open file \"./tests/samples/bad-path\"")]
    fn new_profile_from_file_bad_path(){
		let mut profile = Profile::from_file("./tests/samples/bad-path");
    	profile.pre_generate();

    	assert!(profile.generate().len() > 0);
    }

	#[test]
    fn new_profile_from_serialized(){
    	let serialized = "{\"patterns\":{\"VC\":1},\"pattern_total\":1,\"pattern_keys\":[\"VC\"],\"pattern_vals\":[1],\"pattern_percentages\":[],\"pattern_ranks\":[],\"sizes\":{\"2\":1},\"size_total\":1,\"size_ranks\":[],\"processors\":4,\"facts\":[[{\"key\":\"O\",\"prior_key\":null,\"next_key\":\"K\",\"pattern_placeholder\":\"V\",\"starts_with\":1,\"ends_with\":0,\"index_offset\":0}],[{\"key\":\"K\",\"prior_key\":\"O\",\"next_key\":null,\"pattern_placeholder\":\"C\",\"starts_with\":0,\"ends_with\":1,\"index_offset\":1}],[],[]]}";
    	let mut profile = Profile::from_serialized(&serialized);
    	profile.pre_generate();

    	assert_eq!(profile.generate(), "OK");
    }

    #[test]
    fn new_profile_new_with(){
		let profile =  Profile::new_with_processors(10);

        assert_eq!(profile.processors, 10);
	}

    #[test]
    // ensure Profile is analyzing all the sample data points
    fn profile_analyze(){
    	let mut profil =  Profile::new();
    	profil.analyze("Smith, John");
    	profil.analyze("O'Brian, Henny");
    	profil.analyze("Dale, Danny");
    	profil.analyze("Rickets, Ronney");

    	assert_eq!(profil.patterns.len(), 4);
    }

    #[test]
    // ensure Profile is able to find the facts that relate to a pattern
    // NOTE: Dates need work! e.g.: 00/15/0027
    fn profile_generate_from_pattern_date(){
    	let mut profil =  Profile::new();
    	profil.analyze("01/13/2017");
    	profil.analyze("11/24/2017");
    	profil.analyze("08/05/2017");

    	profil.pre_generate();
    	let generated = profil.generate_from_pattern("##p##p####".to_string());

    	assert_eq!(10, generated.len());
    }

    #[test]
    // ensure Profile is able to find the facts that relate to a pattern
    fn profile_generate_from_pattern_string(){
    	let mut profil =  Profile::new();
    	profil.analyze("First");
    	profil.analyze("Next");
    	profil.analyze("Last");

    	profil.pre_generate();
    	let generated = profil.generate_from_pattern("Cvcc".to_string());

    	assert_eq!(4, generated.len());
    }

    #[test]
    // ensure Profile is generating correct test data
    fn profile_generate(){
    	let mut profil =  Profile::new();
    	profil.analyze("Smith, John");
    	profil.analyze("O'Brian, Henny");
    	profil.analyze("Dale, Danny");
    	profil.analyze("Rickets, Ronnae");
    	profil.analyze("Richard, Richie");
    	profil.analyze("Roberts, Blake");
    	profil.analyze("Conways, Sephen");

    	profil.pre_generate();

    	assert!(profil.generate().len() > 10);
    }

    #[test]
    // issue #31
    // ensure Profile doesn't generate a name with a backslash preceding an apostrophe
    fn profile_generate_with_apostrophe(){
    	let mut profil =  Profile::new();
    	profil.analyze("O'Brien");

    	profil.pre_generate();
    	let generated = profil.generate();

    	assert_eq!(generated, "O'Brien");
    }

    #[test]
    // ensure Profile is providing the correct pattern ranks after analyzing the sample data
    fn profile_pregenerate_patterns(){
    	let mut profil =  Profile::new();
    	profil.analyze("Smith, John");
    	profil.analyze("O'Brian, Henny");
    	profil.analyze("Dale, Danny");
    	profil.analyze("Rickets, Ronnae");
    	profil.analyze("Richard, Richie");
    	profil.analyze("Roberts, Blake");
    	profil.analyze("Conways, Sephen");

    	profil.pre_generate();
    	let test = [("CvccvccpSCvccvv".to_string(), 28.57142857142857 as f64), ("CcvccpSCvcc".to_string(), 42.857142857142854 as f64), ("CvccvccpSCvccvc".to_string(), 57.14285714285714 as f64), ("CvcvcccpSCcvcv".to_string(), 71.42857142857142 as f64), ("CvcvpSCvccc".to_string(), 85.7142857142857 as f64), ("V@CcvvcpSCvccc".to_string(), 99.99999999999997 as f64)];

    	assert_eq!(profil.pattern_ranks, test);
    }

    #[test]
    // ensure Profile is providing the correct pattern ranks after analyzing the sample data
    fn profile_pregenerate_sizes(){
    	let mut profil =  Profile::new();

    	profil.analyze("Smith, Johny"); //12
    	profil.analyze("O'Brian, Hen"); //12
    	profil.analyze("Dale, Danny");  //11
    	profil.analyze("O'Henry, Al");  //11
    	profil.analyze("Rickets, Ro");  //11
    	profil.analyze("Mr. Wilbers");  //11
    	profil.analyze("Po, Al");       //6

    	profil.pre_generate();
    	let test = [(11, 57.14285714285714), (12, 85.71428571428571), (6, 100 as f64)];

    	assert_eq!(profil.size_ranks, test);
    }

	#[test]
    fn save_profile(){
		let mut profile =  Profile::new();
		profile.analyze("Smith, John");
    	profile.analyze("O'Brian, Henny");
    	profile.analyze("Dale, Danny");
    	profile.analyze("Rickets, Ronney");

    	profile.pre_generate();

        assert_eq!(profile.save("./tests/samples/sample-00-profile").unwrap(), true);
	}

    #[test]
    // ensure a Profile can be exported (to be archived) as JSON
    fn serialize(){
		let mut profil =  Profile::new();

		// analyze the dataset
		profil.analyze("OK");

    	let serialized = profil.serialize();
    	assert_eq!(serialized, "{\"id\":null,\"patterns\":{\"VC\":1},\"pattern_total\":1,\"pattern_keys\":[\"VC\"],\"pattern_vals\":[1],\"pattern_percentages\":[],\"pattern_ranks\":[],\"sizes\":{\"2\":1},\"size_total\":1,\"size_ranks\":[],\"processors\":4,\"facts\":[[{\"key\":\"O\",\"prior_key\":null,\"next_key\":\"K\",\"pattern_placeholder\":\"V\",\"starts_with\":1,\"ends_with\":0,\"index_offset\":0}],[{\"key\":\"K\",\"prior_key\":\"O\",\"next_key\":null,\"pattern_placeholder\":\"C\",\"starts_with\":0,\"ends_with\":1,\"index_offset\":1}],[],[]]}");
    }
}
