diff --git a/.cache b/.cache new file mode 100644 index 0000000..0122c72 Binary files /dev/null and b/.cache differ diff --git a/.classpath b/.classpath new file mode 100644 index 0000000..2a3de4b --- /dev/null +++ b/.classpath @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/.project b/.project new file mode 100644 index 0000000..13cd21c --- /dev/null +++ b/.project @@ -0,0 +1,18 @@ + + + PapersProject + + + + + + org.scala-ide.sdt.core.scalabuilder + + + + + + org.scala-ide.sdt.core.scalanature + org.eclipse.jdt.core.javanature + + diff --git a/README.txt b/README.txt new file mode 100644 index 0000000..2494e3c --- /dev/null +++ b/README.txt @@ -0,0 +1,15 @@ +COMPILATION + +In order to correctly compile and execute Trail Head, use sbt or anything you want. +The only important thing is that the repository containing the final executable file must also contain: +- the "tools" dossier +- the "cache" dossier +- the "pdf2xml.dtd" file + + + +EXECUTION + +When you start the program, after providing the input path, if at that location there isn't any "schedule.xml", an exception will be rised. Since this wasn't part of my tasksand I don't exactly know how this works, I didn't touch at that code part. +However, the parsing has been completed before the exception. +If you don't want this annoying exception, just comment the code after the parsing process. \ No newline at end of file diff --git a/build.sbt b/build.sbt index c5f05a7..e630fad 100644 --- a/build.sbt +++ b/build.sbt @@ -2,12 +2,24 @@ name := "Parsing papers" version := "1.0" -scalaVersion := "2.9.1" +scalaVersion := "2.9.2" scalaSource in Compile <<= baseDirectory(_ / "src/paper") scalacOptions ++= Seq("-unchecked", "-Ywarn-dead-code", "-deprecation") +libraryDependencies ++= Seq( + // other dependencies here + // pick and choose: + "org.scalanlp" %% "breeze-process" % "0.1" +) + +resolvers ++= Seq( + // other resolvers here + // if you want to use snapshot builds (currently 0.2-SNAPSHOT), use this. + "Sonatype Snapshots" at "https://oss.sonatype.org/content/repositories/snapshots/" +) + initialCommands := """ import System.{currentTimeMillis => now} def time[T](f: => T): T = { diff --git a/pdf2xml.dtd b/pdf2xml.dtd new file mode 100644 index 0000000..f9caef1 --- /dev/null +++ b/pdf2xml.dtd @@ -0,0 +1,28 @@ + + + + + + + + + + diff --git a/src/paper/Analyze.scala b/src/paper/Analyze.scala index 851c898..99f9206 100644 --- a/src/paper/Analyze.scala +++ b/src/paper/Analyze.scala @@ -1,23 +1,71 @@ package paper object Analyze { - def main(args : Array[String]): Unit = { - - // create analyse + def main(args : Array[String]): Unit= { + // create analyzer val A : Analyzer = new Analyzer() // Check that a directory is supplied (there is an argument) - if (args.length == 0) println("You really need to supply a directory as argument"); + if (args.length == 0 || args.length > 2) { + println("You should provide at least a path and at most a path and an option. Type -h for help."); + } + + else { + val options = readOptions(args.last) - // Then go ahead - else A.analyze(args(0)) + // Then go ahead + A.analyze(args(0), options) + } + } + + // Reads in the options and converts them to a map of options + def readOptions(s : String) : Map[String,Boolean] = { + + var options : Map[String,Boolean] = Map(); + + if (s.length > 0 && s(0) == '-') { + // Define options as always false, except + options = Map().withDefaultValue(false) + + // Check for parsing + if (s.contains('p')) options += ("parse" -> true) + + // Check for getting schedule + if (s.contains('s')) options += ("xmlschedule" -> true) + + // Check for extending + if (s.contains('e')) options += ("extend" -> true) + + // Check for linking + if (s.contains('l')) options += ("link" -> true) + + // Check for linking + if (s.contains('g')) options += ("graph" -> true) + + if(s.contains('h')) { + println("""How to call: Analyze [path] + [parameter]?\nPARAMETERS:\n\t-p : parsing\n\t-s : looks for xml + scheduler\n\t-c : compare\n\t-l : link\n\t-g : create graph\n\t-h + : shows this help page\n\tnothing : do everything"""); + } + + } + + else { + // If no options are supplied we do everything by default + options = Map().withDefaultValue(true) + } + + return options } } + + class Analyzer extends Object with LoadPaper with ParsePaper with ExtendPaper - with ComparePaper + with BagOfWordsLSI with XMLScheduleParser with Graphs { @@ -25,32 +73,46 @@ class Analyzer extends Object with LoadPaper val limit : Int = 1 // Get cached papers in this order - val cache : List[String] = List(Cache.extended, Cache.linked, Cache.scheduled, Cache.parsed) + val cache : List[String] = List(Cache.linked, Cache.extended, Cache.scheduled, Cache.parsed) // Set sources we want to extend with //val sources : List[PaperSource] = List(TalkDates, TalkRooms, PdfLink) val sources : List[PaperSource] = List(PdfLink) // Analyze a paper - def analyze(paperPos: String): Unit = { + def analyze(paperPos: String, options: Map[String, Boolean]): List[Paper] = { + + var papers : List[Paper] = List(); // Get a list of parsed papers - val papers : List[Paper] = load(paperPos, cache, Isit) + if (options("parse") == true) { + papers = loadAndParse(paperPos, cache, XMLParser, XMLConverterLoader) + } // Mix in the schedule XML data - val xmlPapers : List[Paper] = getXMLSchedule(paperPos, papers) + if (options("xmlschedule") == true) { + papers = getXMLSchedule(paperPos, papers) + } + + // Extend papers with tertiary data + if (options("extend") == true) { + papers = extend(paperPos, papers, sources) + } // Compare the papers individually - val comparedPapers : List[Paper] = compare(xmlPapers, limit) + if (options("link") == true) { + papers = compareBoWLSI(paperPos, papers, limit) + } - // Extend papers with tertiary data - val extendedPapers : List[Paper] = extend(comparedPapers, sources) - // Create graph - val graph : Graph = getGraph(extendedPapers) + if (options("graph") == true) { + val graph : Graph = getGraph(paperPos, papers) - // Print graph to file 'data.json' - graph.save + // Print graph to file 'data.json' + graph.save + } + // Now return the papers as is + return papers } } diff --git a/src/paper/BagOfWords.scala b/src/paper/BagOfWords.scala new file mode 100644 index 0000000..d32e602 --- /dev/null +++ b/src/paper/BagOfWords.scala @@ -0,0 +1,268 @@ +package paper +import scala.collection.immutable.List + + + trait BagOfWords { + + + //compare based on scores and return List[Paper] + def compareBoW(paperPos: String, papers : List[Paper], limit : Int) : List[Paper] = { + val loadedPapers = if(papers == List()) CacheLoader.load(paperPos, Cache.extended) else papers + val matrixOfWeights: Array[Array[Int]] = getMatrixOfScores(loadedPapers) + loadedPapers.map(p => { + // Check that paper isn't already linked + if (p.meta.get("linked") == None) { + println("Getting linked") + // Get list of papers that aren't current paper + val otherPapers = loadedPapers.filter(p != _) + + // Compare to every other paper + // Test + val weights : List[Int] = for (other <- otherPapers) yield getScores(matrixOfWeights, p.index)(other.index) + println("weights: " + weights.mkString(", ")) + // Make links + //val links = for ((p,w) <- otherPapers.zip(weights) if w >= limit) yield Link(p.id,w) + val links = for ((p,w) <- otherPapers.zip(weights) if w >= limit) yield Link(p.id,w) + println(links) + + // Add links to paper, and set it as linked + val result = p.setLinks(links).setMeta("linked", "yes") + + // Save result + Cache.save(result, Cache.linked) + + result + } + else p + }) + } + + def getScores(matrixOfScores: Array[Array[Int]], column: Int): List[Int] ={ + + val matrixOfScoresTranspose = matrixOfScores.transpose + + return matrixOfScoresTranspose(column).toList + + } + def getMatrixOfScores(papers: List[Paper]): Array[Array[Int]] ={ + + //val filesList = new java.io.File(directory.toString).listFiles.filter(_.getName.endsWith(".txt")) + + val datasetSize = papers.length + //convert dataset size to float to avoid errors + datasetSize.toFloat + + //Initialisation of arrays + //Array storing the different sources and the different texts + val source = new Array[scala.io.BufferedSource](papers.length) + //we will be changing the content of text -> create it as variable + var text = new Array[java.lang.String](papers.length) + + val occurences = new Array[Map[java.lang.String,Array[java.lang.String]]](papers.length) + //now we want to have a map between words and the number of occurences + //create an array for easier manipulation + val counts = new Array[Map[java.lang.String,Int]](papers.length) + + //Create an array of lists to store all different lists of keys: + val countsList = new Array[List[java.lang.String]](papers.length) + + //List holding all the list of strings of all the texts + var textsList = List[java.lang.String]() + //reading from every entry of the list: + for (k <- 0 to papers.length-1){ + + //source(k) = scala.io.Source.fromFile(papers(k)) + //text(k) = source(k).mkString + text(k) = papers(k).getAbstract.getText + //leave out unecessary characters from the analysis + text(k) = clean(text(k)) + //Splitting the string into words to add stemming to every single word + val splitString = text(k).split("\\s") + var stemmedString = new Array[java.lang.String](splitString.length) + var i = 0 + splitString foreach {e=> + val a = breeze.text.analyze.PorterStemmer.apply(e) + //There is still a blank space at the beginning of string (does not affect output) + stemmedString(i) = a + i+=1 + } + //source(k).close() + counts(k) = stemmedString.groupBy(x=>x).mapValues(x=>x.length) + + //counts(k) foreach{e=> + // val a = clean(e._1) + // newCounts(k).+((a,e._2))} + //only working with keys for now, creating a list of keys for every text: + countsList(k) = counts(k).keys.toList + + if(k == 0){ + textsList = countsList(k) + + }else{ + textsList = textsList ::: countsList(k) + + } + } + + //println(counts.deep.mkString("\n")) + //building dictionary: + //find unique words in texts: + //val texts = textsList.flatten + //var newtextsList = List[java.lang.String]() + //textsList foreach {e => val a = breeze.text.analyze.PorterStemmer.apply(e) + //newtextsList = newtextsList ::: List(a)} + //val textsLength = newtextsList.length + val dictionary = textsList.distinct.sortWith(_<_) + //println(dictionary) + + // we compute the array of scores for the vectors of words for every document + val tfidfArray = new Array[Array[Double]](dictionary.length,datasetSize) + + println("Computing tfidf array... ") + for (i <- 0 to dictionary.length -1){ + for (j <- 0 to datasetSize -1){ + //compute tfidf value for word i and document j + tfidfArray(i)(j) = tfidf(dictionary(i),j,datasetSize,counts) + //println(tfidfArray(i)(j)) + } + } + println("Computing tfidf array: Complete...") + //once we have the scores we can compute the absolute distance between papers and classify them + //This is performed computing a scalar product on the score vectors for every document + //Computation might take some time + + //temporary while getting "scalala" to work: + val scalarProduct = new Array[Array[Double]](datasetSize,datasetSize) + val cosineSimilarity = new Array[Array[Double]](datasetSize,datasetSize) + //transpose array to perform row Array operations instead of column based operations + //println(tfidfArray.deep.mkString("\n")) + val tfidfTranspose = tfidfArray.transpose + + val normalisationTerm = 1 + println("Computing scalar product array") + for (i <- 0 to datasetSize -1){ + //println(i) + for (j <- 0 to datasetSize -1){ + if(i==j) + cosineSimilarity(i)(j) = 0 + else{ + //May induce some computational time + val normVectorI = math.sqrt(dotProduct(tfidfTranspose(i),tfidfTranspose(i))) + val normVectorJ = math.sqrt(dotProduct(tfidfTranspose(j),tfidfTranspose(j))) + //println("dot product of " + tfidfTranspose(i).deep.mkString("\n") + " and " + tfidfTranspose(j).deep.mkString("\n") + " is " + dotProduct(tfidfTranspose(i),tfidfTranspose(j))) + + //Here operations take cost of length O(dictionary length) + //compute cosine similarity + scalarProduct(i)(j) = dotProduct(tfidfTranspose(i), tfidfTranspose(j)) + //println(" " + normVectorI + " " + tfidfTranspose(i).deep.mkString("\n")) + //println(scalarProduct(i)(j)) + cosineSimilarity(i)(j) = scalarProduct(i)(j)/(normVectorI*normVectorJ) + //println(cosineSimilarity(i)(j) + " " + scalarProduct(i)(j) + " " + normVectorI + " " + normVectorJ) + + } + } + } + //return array of scores + println("Computing scalar product array: Done...") + // map every score with the paper ID + //for every paper sort according to scores + println("Sorting accordingly...") + val a = 0 until datasetSize + var positions = new Array[List[(Double,Int)]](datasetSize) + for(k <- 0 to datasetSize-1){ + positions(k) = (scalarProduct(k).zip(a)).toList.sortWith(_._1 < _._1 ) + } + //check if rounding is done correctly + val maximalWeight = cosineSimilarity.flatten.max + //println(maximalWeight) + val normalizedCosSimilarity = cosineSimilarity.map(col =>{ + col.map(weight => ((weight*100)/maximalWeight).toInt) + }) + //return sorted scores with according paper + //println(cosineSimilarity.deep.mkString("\n")) + + normalizedCosSimilarity + //(i,j) of scalarProduct represents the scalar product of document i and document j. Now we have + // to sort it in order in a list to return the closest documents to a given document + //we have weights (higher weight/score) means being closer document-to-document wise + } + + + + // Code has to be made generic for any text file parsed and for the whole dataset to be accurate + + //reading text from given file + //Loading the List of all available text files in directory + + + +// working with lists: + + + + //building dictionnary: + //find unique words in texts: + + + //println("The total length of the dictionnary is given by: " + dictionary.length) + + //println("the total length of the list is: " + textsLength) + + //Computing TF value: + + def tf(term: String, document: Int, counts: Array[Map[java.lang.String,Int]]): Double = { + //Without normalisation + if (counts(document).contains(term)){ + val freq = counts(document)(term) + val normalizedFreq = freq + return normalizedFreq + }else{ + return 0.0 + } + //normalization with respect to the documents length to prevent any bias: + //new normalisation with respect to the highest occurence in the document + + } + + //Computing IDF value + + def idf(term: String, datasetSize : Double, counts: Array[Map[java.lang.String,Int]]): Double = { + //math.log(size / index.getDocCount(term)) + // take the logarithm of the quotient of the number of documents by the documents where term t appears + var appearances = 0 + //convert appearances to a float (to avoid errors) + appearances.toFloat + //println(counts.deep.mkString("\n")) + counts foreach {x => if (x.contains(term)){ + appearances += 1 + + } + //println(term + " => appearances: " + appearances) + } + val a = math.log(datasetSize/appearances) + return a + + } + + def tfidf(term:String, document: Int, datasetSize : Double, counts: Array[Map[java.lang.String,Int]]) : Double = { + //create tfidf matrix + //tfidf = tf*idf + + val tfidf = tf(term,document,counts)*idf(term,datasetSize,counts) + //println("For document " + document + " and word " + term + " the value of tf is " + tf(term,document,counts) + " and the value of idf is " + idf(term,datasetSize,counts)) + return tfidf + + } + + //defining scala product for array vector operations + def dotProduct[T <% Double](as: Iterable[T], bs: Iterable[T]) = { + require(as.size == bs.size) + (for ((a, b) <- as zip bs) yield a * b) sum + } + + //replace all characters of a string except for a-z or A-Z (replacing numbers) and finally _: + def clean(in : String) = { if (in == null) "" else in.replaceAll("[^a-zA-Z_]", " ").toLowerCase + } + + +} diff --git a/src/paper/BagOfWordsLSI.scala b/src/paper/BagOfWordsLSI.scala new file mode 100644 index 0000000..c77eeec --- /dev/null +++ b/src/paper/BagOfWordsLSI.scala @@ -0,0 +1,600 @@ +package paper + + +import breeze.linalg.DenseVector +import breeze.classify +import org.netlib.lapack.LAPACK +import org.netlib.util.intW +import breeze.linalg.support.{CanCopy} + + + +trait bagOfWordsLSI { + + + // error is here: + def compareBoWLSI(paperPos: String, papers : Option[List[Paper]], limit : Int) : List[Paper] = { + val loadedPapers = if(papers == None) CacheLoader.load(paperPos, Cache.extended) else papers.get + val matrixOfWeights: breeze.linalg.DenseMatrix[Int] = createTDMatrix(loadedPapers,loadedPapers.length) + loadedPapers.map(p => { + // Check that paper isn't already linked + if (p.meta.get("linked") == None) { + // Get list of papers that aren't current paper + val otherPapers = loadedPapers.filter(p != _) + println(getScores(matrixOfWeights, p.index).toString) + // Compare to every other paper + // Problem is in this line + val weights : List[Int] = for (other <- otherPapers) yield getScores(matrixOfWeights,p.index).valueAt(other.index) + // Make links + //val links = for ((p,w) <- otherPapers.zip(weights) if w >= limit) yield Link(p.id,w) + val links = for ((p,w) <- otherPapers.zip(weights) if w >= limit) yield Link(p.id,w) + + // Add links to paper, and set it as linked + val result = p.setLinks(links).setMeta("linked", "yes") + + // Save result + Cache.save(result, Cache.linked) + + result + } + else p + }) + } + + def getScores(matrixOfScores: breeze.linalg.DenseMatrix[Int], column: Int): DenseVector[Int] ={ + + val matrixOfScoresTranspose = matrixOfScores.t + + return matrixOfScoresTranspose(::,column) + + } + + //Methods to compute the term-document matrix + def tf(term: String, document: Int, counts: Array[Map[java.lang.String,Int]]): Double = { + val keyValue = counts(document).values + val normalisationTerm = keyValue.max + //test without normalisation + if (counts(document).contains(term)){ + val freq = counts(document)(term) + val normalizedFreq = freq + ///normalisationTerm + return normalizedFreq + }else{ + return 0.0 + } + } + + //Computing IDF value + def idf(term: String, datasetSize : Double, counts: Array[Map[java.lang.String,Int]]): Double = { + //math.log(size / index.getDocCount(term)) + // take the logarithm of the quotient of the number of documents by the documents where term t appears + var appearances = 0 + //convert appearances to a float (to avoid errors) + appearances.toFloat + //println(counts.deep.mkString("\n")) + counts foreach {x => if (x.contains(term)){ + appearances += 1 + + } + //println(term + " => appearances: " + appearances) + } + val a = math.log(datasetSize/appearances) + return a + } + + def tfidf(term:String, document: Int, datasetSize : Double, counts: Array[Map[java.lang.String,Int]]) : Double = { + //Create tfidf matrix + //tfidf = tf*idf + val tfidf = tf(term,document,counts)*idf(term,datasetSize,counts) + return tfidf + } + + //Creating the matrix: + def createTDMatrix(papers: List[Paper], approximation: Int): breeze.linalg.DenseMatrix[Int] = { + val datasetSize = papers.length + //val filesList = new java.io.File(directory.toString).listFiles.filter(_.getName.endsWith(".txt")) + //val datasetSize = filesList.length + datasetSize.toFloat + //Initialisation of arrays + //Array storing the different sources and the different texts + val source = new Array[scala.io.BufferedSource](papers.length) + var text = new Array[java.lang.String](papers.length) + val occurences = new Array[Map[java.lang.String,Array[java.lang.String]]](papers.length) + //now we want to have a map between words and the number of occurences + //create an array for easier manipulation + var counts = new Array[Map[java.lang.String,Int]](papers.length) + var newCounts = new Array[Map[java.lang.String,Int]](papers.length) + //Create an array of lists to store all different lists of keys: + val countsList = new Array[List[java.lang.String]](papers.length) + //List holding all the list of strings of all the texts + var textsList = List[java.lang.String]() + //reading from every entry of the list: + + for (k <- 0 to papers.length-1){ + //source(k) = scala.io.Source.fromFile(filesList(k)) + //text(k) = source(k).mkString + text(k) = papers(k).getAbstract.getText + //leave out unecessary characters from the analysis + text(k) = clean(text(k)) + //Splitting the string into words to add stemming to every single word + val splitString = text(k).split("\\s") + var stemmedString = new Array[java.lang.String](splitString.length) + var i = 0 + splitString foreach {e=> + val a = breeze.text.analyze.PorterStemmer.apply(e) + //There is still a blank space at the beginning of string (does not affect output) + stemmedString(i) = a + i+=1 + } + + //source(k).close() + //Array of the words of the parsed text + //words = clean(words) + //var stemmedWords = new Array[java.lang.String](words.length) + //Using stemmer for every text: + //words foreach{e => + //stemmedWords(counter) = breeze.text.analyze.PorterStemmer.apply(e) + //counter += 1 + //} + //occurences(k) = stemmedWords.groupBy(x=>x) + // create a map of the keys of the text with their occurences + //the problem was the split... + counts(k) = stemmedString.groupBy(x=>x).mapValues(x=>x.length) + //counts(k) foreach{e=> + // val a = clean(e._1) + // newCounts(k).+((a,e._2))} + //only working with keys for now, creating a list of keys for every text: + countsList(k) = counts(k).keys.toList + + + if(k == 0){ + textsList = countsList(k) + }else{ + textsList = textsList ::: countsList(k) + } + } + + //adding stemming to the text + //var newtextsList = List[java.lang.String]() + // textsList foreach {e => val a = breeze.text.analyze.PorterStemmer.apply(e) + // newtextsList = newtextsList ::: List(a)} + //building dictionary: + //find unique words in texts + val textsLength = textsList.length + val dictionary = textsList.removeDuplicates.sort(_<_) + println(dictionary) + // we compute the Matrix of scores for the vectors of words for every document + //construct it as a vector to convert it as a Matrix + val tfidfVector = new Array[Double](dictionary.length*datasetSize) + var j = 0 + println("Computing tfidf vector... with a dictionary of length " + dictionary.length + " and wait up to " + dictionary.length*datasetSize) + for (i <- 0 to dictionary.length*datasetSize-1){ + println(i) + //compute tfidf value for word i and document j + //check if we have reached the length of the dictionary we change document and compute values + if (i % dictionary.length == 0 && i != 0){ + j += 1 + } + + tfidfVector(i) = tfidf(dictionary(i%dictionary.length),j,datasetSize,counts) + } + + println("Computing tfidf vector: Complete...") + + val termDocMatrix = new breeze.linalg.DenseMatrix[Double](dictionary.length,datasetSize,tfidfVector) + println(termDocMatrix.toString) + println("done") + //once having the termDocMatrix, compute the SVD and then compute cosine similarity on the new matrix: + + + //SVD method returns simple arrays - need to convert them: + val (u,s,v) = svd(termDocMatrix) + //println(u.deep.mkString("\n")) + //println("end of u") + println(s.deep.mkString("\n")) + println("end of s") + //println(v.deep.mkString("\n")) + //println("end of v") + + //converting w and d to 2 dimensional arrays: + var uo = reconstructArray(u,termDocMatrix.rows) + println(uo.length + " " + uo.transpose.length) + var vo = reconstructArray(v,termDocMatrix.cols) + println(vo.length + " " + vo.transpose.length) + var so = Array.ofDim[Double](termDocMatrix.rows,termDocMatrix.cols) + println(so.length + " " + so.transpose.length) + var count = 0 + var count2 = 0 + + // put the vector s into a n*m matrix so form: + s foreach{e=> + if(count2 <= termDocMatrix.cols-1){ + so(count)(count2) = e + count += 1 + count2 += 1 + } + } + + val test = multiplyArrays(uo,multiplyArrays(so,vo)) + val newtest = new Array[Array[Double]](10,10) + for(i<- 0 to 9){ + for (j<- 0 to 9){ + newtest(i)(j)=so(i)(j) + } + } + //printing s to see the values + //println(s.deep.mkString("\n")) + + //keeping k greatest singular values: + //test with half the length of the vector + val emptyList = List[Int]() + println("computing k maximal values") + //need to make a local copy of s: + //val so = copy(s) + // SVD returns the first k values as the k highest values - assuming no need for indices: + //val indices = findKMax(s, approximation, emptyList) + + println("computed k maximal values") + //println(indices) + //select values of the indices in the given array s - indices are the 0 to k-1 first values: + val emptyArray = new Array[Array[Double]](0) + val keptValues = selectElementsOf2dimArray(so,(0 to approximation-1).toList,emptyArray) + var newKeptValues = new Array[Array[Double]](keptValues.length,keptValues.length) + var arrayCounter = 0 + + println(keptValues.deep.mkString("\n")) + //keptValues.transpose foreach { x=> arrayCounter match{ + //case y if arrayCounter < indices.length => newKeptValues(arrayCounter) = x + //case y if arrayCounter >= indices.length => x + //arrayCounter += 1 + //} + val keptVTr = keptValues.transpose + keptVTr foreach{e => + if (arrayCounter <= approximation-1){ + newKeptValues(arrayCounter) = e + arrayCounter += 1 + }else{ + Nil + } + } + println("separation") + println(newKeptValues.deep.mkString("\n")) + + //matrix multiplication: + //keep the same values for the matrix multiplication: XO = UO*SO*VtO + val emptyArray2 = new Array[Array[Double]](0) + val emptyArray3 = new Array[Array[Double]](0) + var newUo = selectElementsOf2dimArray(uo.transpose,(0 to approximation-1).toList,emptyArray2) + newUo = newUo.transpose + var newVo = selectElementsOf2dimArray(vo,(0 to approximation-1).toList,emptyArray3) + newVo = newVo.transpose + //Multiplication: + println(newUo.length,newUo.transpose.length,keptValues.length,keptValues.transpose.length,newVo.length,newVo.transpose.length) + println(keptValues.length + " " + keptValues.transpose.length + " " + newVo.length + " " + newVo.transpose.length) + val xo = multiplyArrays(newUo,multiplyArrays(newKeptValues,newVo)) + + + //currently performing test without newUo or newVo + // Obtain a single dimension array out of the resulting array xo + //var recomposedMatrix = new Array[Double](newUo.length*newVo.transpose.length) + //var k = 0 + //useless - use flatten method instead: + //xo.transpose foreach { e => e foreach{ b=> recomposedMatrix(k) = b + // k= k+1 + // if(k-1<=20){ + // println(recomposedMatrix(k))} + //} + //} + //replaced xo by test + val recomposedMatrix = xo.transpose.flatten + val newrecomposedMatrix = test.transpose.flatten + var counter = 0 + + recomposedMatrix foreach{ e => + if(counter<=30) {println("recomposed -> " + e) + println("tfidf -> " + tfidfVector(counter)) + println("newrecomp -> " + newrecomposedMatrix(counter))} + counter+=1} + //println(recomposedMatrix.deep.mkString("\n")) + + // Compute distance between the two matrices and then compute the norm of the average distance + + //create an iterator that goes through the array: + //Edit: does not work because termDocMatrix is a Matrix and not an array as per + //Iterator.fromArray(xo) foreach (x => Iterator.fromArray(termDocMatrix) foreach (y => x.zip(y))) + + def matchApproximation(app: Int): breeze.linalg.DenseMatrix[Double] = app match{ + case termDocMatrix.cols => new breeze.linalg.DenseMatrix[Double](termDocMatrix.rows,termDocMatrix.cols,newrecomposedMatrix) + + case 0 => new breeze.linalg.DenseMatrix[Double](termDocMatrix.rows,termDocMatrix.cols,newrecomposedMatrix) + + case y => new breeze.linalg.DenseMatrix[Double](termDocMatrix.rows,termDocMatrix.cols,recomposedMatrix) + } + + val newtermDocMatrix = matchApproximation(approximation) + val distanceMat = Math.sqrt((newrecomposedMatrix.toList.zip(tfidfVector.toList) foldLeft 0.0)((sum,t) => + sum + (t._2-t._1)*(t._2-t._1))) + //compute cosine similarity: + //println(newtermDocMatrix.toString) + val similarityMatrix = breeze.linalg.DenseMatrix.zeros[Double](datasetSize,datasetSize) + + //not optimal version: + for (i <- 0 to datasetSize-1){ + for (j <- 0 to datasetSize-1){ + if(i==j){ + similarityMatrix(i,j) = -1 + }else{ + //Compute scalar product between two matrices + // maybe perform transpose? + val firstColumn = newtermDocMatrix(0 to newtermDocMatrix.rows-1,i) + + val secondColumn = newtermDocMatrix(0 to newtermDocMatrix.rows-1,j) + //println(firstColumn.toString,i) + similarityMatrix(i,j) = firstColumn.dot(secondColumn) + + //Compute 2nd norm and output cosine similarity + val firstColumnNorm = firstColumn.norm(2) + //println(firstColumnNorm) + val secondColumnNorm = secondColumn.norm(2) + //test without normalisation: + similarityMatrix(i,j) = similarityMatrix(i,j)/(firstColumnNorm*secondColumnNorm) + println(i + " and j is " + j ) + println(similarityMatrix(i,j)) + } + } + } + val maximalWeight = similarityMatrix.max + //val smat = similarityMatrix.toString + //1 way to do it + println(distanceMat) + val normalizedCosSimilarity = similarityMatrix.map(weight =>{ + ((weight*100)/maximalWeight).toInt + }) + + return normalizedCosSimilarity + //Code works but there are several problems: NaN for some similarity values... not normal + // NaN values are due to the division of 0 by a really high number... Need to perform tests. + } + + /* +def getScores(matrixOfScores: DenseMatrix[Double], column: Int): List[Double] ={ + + //val matrixOfScoresTranspose = matrixOfScores.transpose + + // return matrixOfScoresTranspose(column).toList + +} + */ + def exportMatrixToText(matrix: String) : Unit = { + val file = new java.io.File("exportToGephi.csv") + val p = new java.io.PrintWriter(file) + p.println(matrix) + p.close + } + //remove an element on a given index from a given list: + def dropIndex[T](xs: List[T], n: Int) = { + val (l1, l2) = xs splitAt n + l1 ::: (l2 drop 1) + } + + def selectElementsOfArray(inputArray: Array[Double], inputIndices: List[Int], returnArray: Array[Double]): Array[Double] = { + //var returnArray = new Array[Double](inputIndices.length) + if(inputIndices == Nil){ + inputArray + }else if(inputIndices.length != 1){ + //Add element of the input array corresponding to the first index in the index list: + var newreturnArray = returnArray :+ inputArray(inputIndices(0)) + selectElementsOfArray(inputArray,dropIndex(inputIndices,0),newreturnArray) + }else{ + var newreturnArray = returnArray :+ inputArray(inputIndices(0)) + return newreturnArray + } + } + + def dotProduct[T <% Double](as: Iterable[T], bs: Iterable[T]) = { + require(as.size == bs.size) + (for ((a, b) <- as zip bs) yield a * b) sum + } + + def selectElementsOf2dimArray(inputArray: Array[Array[Double]], inputIndices: List[Int], returnArray: Array[Array[Double]]): Array[Array[Double]] = { + //var returnArray = new Array[Double](inputIndices.length) + if(inputIndices == Nil){ + inputArray + }else if(inputIndices.length != 1){ + //Add element of the input array corresponding to the first index in the index list: + var newreturnArray = returnArray :+ inputArray(inputIndices(0)) + selectElementsOf2dimArray(inputArray,dropIndex(inputIndices,0),newreturnArray) + }else{ + var newreturnArray = returnArray :+ inputArray(inputIndices(0)) + return newreturnArray + } + } + + def clean(in : String) ={ if (in == null) "" else in.replaceAll("[^a-zA-Z-']"," ").toLowerCase} + + //computing find method to return the k largest indices of elements in a vector + //temporary / must reduce complexity (perform tests etc...) + //deletes maximal values of original array (to fix). Otherwise works perfectly // + def findKMax(inputVector: Array[Double], k: Int, listOfIndex: List[Int]): List[Int]= { + if(listOfIndex.length < k){ + val maxofArray = inputVector.max + if (maxofArray != 0){ + var newlistOfIndex = listOfIndex:::List(inputVector.findIndexOf(x => x == maxofArray)) + //set maximal value to 0 so it does not get taken into account again + inputVector(inputVector.findIndexOf(x => x == maxofArray)) = 0 + findKMax(inputVector,k, newlistOfIndex) + + }else{ + listOfIndex + } + }else{ + return listOfIndex + } + } + // finds the N maximal values (not the indices) + def topNs(xs: Array[Double], n: Int) = { + var ss = List[Double]() + var min = Double.MaxValue + var len = 0 + xs foreach { e => + if (len < n || e > min) { + ss = (e :: ss).sorted + min = ss.head + len += 1 + } + if (len > n) { + ss = ss.tail + min = ss.head + len -= 1 + } + } + ss + } + + // to do: fix output + def indexOftopNs(xs: Array[Double], n: Int) = { + var ss = List[Int]() + var min = Double.MaxValue + var len = 0 + xs foreach { e => + if (len < n || e > min) { + ss = (xs.findIndexOf(x=>x==e) :: ss).sorted + min = ss.head + len += 1 + } + if (len > n) { + ss = ss.tail + min = ss.head + len -= 1 + } + } + ss + } + + + // function converting a DenseVector to a List of Double + def convertToList(inputVector : DenseVector[Double]) : List[Double] = { + val outputVector = List[Double]() + for(i <- inputVector){ + outputVector:::List(i) + } + return outputVector + } + + //desiredLength = length of the rows (= number of columns) + def reconstructArray(inputArray: Array[Double], desiredLength: Int)={ + val doubleDimArray = Array.ofDim[Double]((inputArray.length/ desiredLength).toInt,desiredLength) + var k = 0 + var i = 0 + inputArray foreach { e => + if(k < desiredLength){ + doubleDimArray(k)(i) = e + k = k+1 + } + if(k == desiredLength){ + k = 0 + i = i+1 + } + } + doubleDimArray + } + //redifining svd: + + //@inline private def requireNonEmptyMatrix[V](mat: Matrix[V]) = + //if (mat.cols == 0 || mat.rows == 0) + //throw new MatrixEmptyException +//modification of the www.netlib.org/lapack/ package, dgesdd method - derived from breeze svd + def svd(mat: breeze.linalg.DenseMatrix[Double]):(Array[Double],Array[Double],Array[Double]) = { + // we do not use the matrix requirements + // requireNonEmptyMatrix(mat) + + val m = mat.rows + val n = mat.cols + //val S = DenseVector.zeros[Double](m min n) + //matrix of zeros + //S = denseVector(UCOL = min(m,n)) + //val S = Matrix(m min n,1){ (i:Int,j:Int) => 0 } + val S = new Array[Double](m min n) + //val U = Matrix(m,m){ (i:Int,j:Int) => 0 } + //val U = breeze.linalg.DenseMatrix.zeros[Double](m,m) + val U = new Array[Double](m*m) + val Vt = new Array[Double](n*n) + //Matrix(n,n){ (i:Int,j:Int) => 0 } + val iwork = new Array[Int](8 * (m min n) ) + val workSize = ( 3 + * scala.math.min(m, n) + * scala.math.min(m, n) + + scala.math.max(scala.math.max(m, n), 4 * scala.math.min(m, n) + * scala.math.min(m, n) + 4 * scala.math.min(m, n)) + ) + val work = new Array[Double](workSize) + val info = new intW(0) + //S.elements.flatten.toArray + //U.take(m*m).flatten.toArray + val cm = copy(mat) + println("im in") + LAPACK.getInstance.dgesdd( + "A", m, n, + cm.data, scala.math.max(1,m), + S, U , scala.math.max(1,m), + Vt, scala.math.max(1,n), + work,work.length,iwork, info) + + if (info.`val` > 0) + throw new NotConvergedException(NotConvergedException.Iterations) + else if (info.`val` < 0) + throw new IllegalArgumentException() + + (U,S,Vt) + } + def copy[T](t: T)(implicit canCopy: CanCopy[T]): T = canCopy(t) + + class MatrixEmptyException extends IllegalArgumentException("Matrix is empty") + + class NotConvergedException(val reason: NotConvergedException.Reason, msg: String = "") + extends RuntimeException(msg) + + object NotConvergedException { + trait Reason + object Iterations extends Reason + object Divergence extends Reason + object Breakdown extends Reason + } + + //Scala for java developers: http://blog.scala4java.com/2011/12/matrix-multiplication-in-scala-single.html + + def multiplyArrays(m1: Array[Array[Double]], m2: Array[Array[Double]]) : Array[Array[Double]] = { + val res = Array.ofDim[Double](m1.length, m2(0).length) + val M1_COLS = m1(0).length + val M1_ROWS = m1.length + val M2_COLS = m2(0).length + + @inline def singleThreadedMultiplicationFAST(start_row:Int, end_row:Int) { + var col, i = 0 + var sum = 0.0 + var row = start_row + + // while statements are much faster than for statements + while(row < end_row){ col = 0 + while(col < M2_COLS){ i = 0; sum = 0 + while(i + singleThreadedMultiplicationFAST(i, i+1) + ) + + res + + } + +} diff --git a/src/paper/ComparePaper.scala b/src/paper/ComparePaper.scala index fabe2cd..0eabc80 100644 --- a/src/paper/ComparePaper.scala +++ b/src/paper/ComparePaper.scala @@ -1,47 +1,51 @@ -package paper -import java.io._ -import scala.io.Source - -trait ComparePaper { - - def compare(papers : List[Paper], limit : Int) : List[Paper] = { - - papers.map(p => { - // Check that paper isn't already linked - if (p.meta.get("linked") == None) { - // Get list of papers that aren't current paper - val otherPapers = papers.filter(p != _) - - // Compare to every other paper - val weights : List[Int] = for (other <- otherPapers) yield getWeight(p, other) - - // Make links - //val links = for ((p,w) <- otherPapers.zip(weights) if w >= limit) yield Link(p.id,w) - val links = for ((p,w) <- otherPapers.zip(weights) if w >= 1) yield Link(p.index,w) - - // Add links to paper, and set it as linked - val result = p.setLinks(links).setMeta("linked", "yes") - - // Save result - Cache.save(result, Cache.linked) - - result - } - else p - }) - } - - def getWeight(p : Paper, o : Paper) : Int = { - // Get names - val pNames = p.getDistinctNames - val oNames = o.getDistinctNames - - // For each auther in p, check if he/she exists in other - var matches = (for (name <- pNames if oNames.contains(name)) yield 1).sum - - // return result - return (100 * matches.toDouble / pNames.length.toDouble).toInt - - } - -} +package paper +import java.io._ +import scala.io.Source + +trait ComparePaper { + + def compare(paperPos:String, papers : Option[List[Paper]], limit : Int) : List[Paper] = { + println("BEGIN OF PAPERS COMPARISION") + val loadedPapers = if(papers == None) CacheLoader.load(paperPos, Cache.extended) else papers.get + + val finalPapers = loadedPapers.map(p => { + // Check that paper isn't already linked + if (p.meta.get("linked") == None) { + // Get list of papers that aren't current paper + val otherPapers = loadedPapers.filter(p != _) + + // Compare to every other paper + val weights : List[Int] = for (other <- otherPapers) yield getWeight(p, other) + + // Make links + //val links = for ((p,w) <- otherPapers.zip(weights) if w >= limit) yield Link(p.id,w) + val links = for ((p,w) <- otherPapers.zip(weights) if w >= 1) yield Link(p.index,w) + + // Add links to paper, and set it as linked + val result = p.setLinks(links).setMeta("linked", "yes") + + // Save result + Cache.save(result, Cache.linked) + + result + } + else p + }) + println("END OF PAPERS COMPARISION") + finalPapers + } + + def getWeight(p : Paper, o : Paper) : Int = { + // Get names + val pNames = p.getDistinctNames + val oNames = o.getDistinctNames + + // For each auther in p, check if he/she exists in other + var matches = (for (name <- pNames if oNames.contains(name)) yield 1).sum + + // return result + return (100 * matches.toDouble / pNames.length.toDouble).toInt + + } + +} diff --git a/src/paper/ExtendPaper.scala b/src/paper/ExtendPaper.scala index d4c170f..420d79c 100644 --- a/src/paper/ExtendPaper.scala +++ b/src/paper/ExtendPaper.scala @@ -43,12 +43,13 @@ object TalkDates extends PaperSource { def getLabel : String = "date" } + +// Adds the link of the pdf to the paper object PdfLink extends PaperSource { import scala.util.Random - // TODO: This is also just a temporary thing def getInfo(p : Paper) : String = { - + var f : String = p.meta("file") var pdf : String = f.takeWhile(_!='.').concat(".pdf") return pdf; @@ -72,16 +73,21 @@ object TalkRooms extends PaperSource { } +/** Extend paper loops through a list of sources. Each source implements the + * interface paperSource and provides two methods: getLabel and getInfo. + * Get label returns the map label, while getInfo returns the particular information + */ trait ExtendPaper { - def extend(papers : List[Paper], sources : List[PaperSource]) : List[Paper] = { - - papers.map(p => { + def extend(paperPos: String, papers : List[Paper], sources : List[PaperSource]) : List[Paper] = { + println("BEGIN OF PAPERS EXTENSION") + val loadedPapers = if(papers == List()) CacheLoader.load(paperPos, Cache.scheduled) else papers + val finalPapers = loadedPapers.map(p => { var result : Paper = p // For each source, check if it's already added, and if not, add it - for (s <- sources if p.hasMeta(s.getLabel)) { + for (s <- sources if !p.hasMeta(s.getLabel)) { result = result.setMeta(s.getLabel, s.getInfo(p)) } @@ -91,6 +97,9 @@ trait ExtendPaper { // return result result }) + + println("END OF PAPERS EXTENSION") + finalPapers } - + } diff --git a/src/paper/FileFormat.scala b/src/paper/FileFormat.scala new file mode 100644 index 0000000..c3423fe --- /dev/null +++ b/src/paper/FileFormat.scala @@ -0,0 +1,72 @@ +package paper +import java.io.File + +object Paths { + private val toolsDirStr = "tools" + private val linuxDirStr = "linux" + private val windowsDirStr = "windows" + private val windowsSepStr = "\\" + private val linuxSepStr = "/" + private val windowsExtStr = ".exe" + private val linuxExtStr = "" + + + private def getOSDir: String = { if(SystemHelper.isWindows) windowsSepStr + windowsDirStr + windowsSepStr + else if(SystemHelper.isLinux) linuxSepStr + linuxDirStr + linuxSepStr + else "" } + + // Returns the tools directory according to the current operating system + def toolsDir = toolsDirStr + getOSDir + + // Returns the file extension according to the current operating system + def ext = if(SystemHelper.isWindows) windowsExtStr else if(SystemHelper.isLinux) linuxExtStr else "" + + // Returns the path separator according to the current operating system + def sep = if(SystemHelper.isWindows) windowsSepStr else if(SystemHelper.isLinux) linuxSepStr else "" +} + +// This class transforms a File object into another File object, according to the extension of the file +abstract class FileFormat { + def convertTo(format: String, params: List[String]): File = this match{ + case TXTFormat(file) => file + case PDFFormat(file) => { + val command = CommandDetector.detect(Paths.toolsDir + "pdfTo" + format + "Converter" + Paths.ext, format) + + val process: Process = sys.runtime.exec((List(command) ::: params ::: List(file.getAbsolutePath())).toArray[String]) + + // Waiting until the end of the command execution + if(process.waitFor() != 0) { println("Can't convert pdf file. Program will exit"); sys.exit } + + new File(file.getParent() + Paths.sep + SystemHelper.name(file.getName) + "." + format) + } + } + + // Performs final procedures before the end of the file processing (mostly deleting intermediate files) + def releaseFile(newFile: File) = this match { + case TXTFormat(file) => + case PDFFormat(file) => newFile.delete() + } +} + +case class TXTFormat (file: File) extends FileFormat +case class PDFFormat (file: File) extends FileFormat + + +object FileFormatDispatcher { + // Determines which FileFormat should be used according to the extension of the file + def getFileFormat(file: File): FileFormat = { + SystemHelper.ext(file.getName()) match { + case "txt" => new TXTFormat(file) + case "pdf" => new PDFFormat(file) + } + } +} + +// This object handles special tool cases +object CommandDetector { + def detect(toolPath: String, format: String): String = { + if(format.equals("xml") && SystemHelper.isLinux) return "pdftohtml" + + toolPath + } +} diff --git a/src/paper/FileLoader.scala b/src/paper/FileLoader.scala new file mode 100644 index 0000000..0ecdef0 --- /dev/null +++ b/src/paper/FileLoader.scala @@ -0,0 +1,112 @@ +package paper +import java.io.File +import scala.io.Source +import scala.io.BufferedSource + +// This object provides helpful methods for file and system managing +object SystemHelper { + private val supportedFormats = """pdf""" + private val os = sys.props.get("os.name") + + def ext(file: String) = """^.+?\.""".r.findFirstIn(file.reverse).get.dropRight(1).reverse + def name(file: String) = """^.+?\.""".r.replaceAllIn(file.reverse, "").reverse + def isSupported(format: String): Boolean = ("^" + supportedFormats + "$").r.findFirstIn(format).isDefined + + // Returns the files from a directory + def getFilesFromDirectory(orig: File): List[File] = { + if(orig.isDirectory()) orig.listFiles.filter(f => (""".+\.(""" + supportedFormats + """)$""").r.findFirstIn(f.getName).isDefined).toList + else Nil + } + + // The apps is on Windows + def isWindows: Boolean = os match { + case Some(s) => """.*Windows.*""".r.findFirstIn(s.toString()).isDefined + case None => false + } + + // The apps is on Linux + def isLinux: Boolean = os match { + case Some(s) => """.*Linux.*""".r.findFirstIn(s.toString()).isDefined + case None => false + } +} + +trait FileLoader { + def loadFromFile(file : File, p : Parsers) : Option[Paper] + + // This function sets the id and metadata before final returning + def setLastModifications(file: File, paper: Option[Paper]): Option[Paper] = { + if (paper != None) { + // Get index + try { + var id = (SystemHelper.name(file.getName())).toInt + + // Set filename and id + val finalPaper : Paper = paper.get.setMeta("file" -> file.getPath).setId(id) + + return Some(finalPaper) + }catch { + case _ => { println("The paper's name must contain only numerical values. Not parsed"); return None } + } + } + return None + } +} + +// This loader doesn't care about the format, it just passes the file to the parser +object SimpleLoader extends FileLoader { + def loadFromFile(file : File, p : Parsers) : Option[Paper] = { + println("parsing " + file.getPath + " using simple loader") + + val text = Source.fromFile(file) + + // actual parsing of the file content + val maybePaper : Option[Paper] = p.parse(text) + + // If paper exists and parsed, save it in cache + setLastModifications(file, maybePaper) + } +} + +// This class uses external tools in order to convert a particular format file into another before it is passed to the parser +abstract class ExternalLoader extends FileLoader { + // Converts a particular format file into another using an external tool + def loadFromFile(file : File, p : Parsers, format : String, params : List[String]) : Option[Paper] = { + // getFileFormat looks for other file formats (like pdf), then using external tools it extracts the content of the file and + // converts it to another format file + val fileFormat = FileFormatDispatcher.getFileFormat(file) + val newFile = fileFormat.convertTo(format, params) + val text = Source.fromFile(newFile) + + // actual parsing of the file content + val maybePaper : Option[Paper] = p.parse(text) + + // this method deletes temporary files that could have been previously created + fileFormat.releaseFile(newFile) + + // If paper exists and parsed, save it in cache + setLastModifications(file, maybePaper) + } +} + +// This object tries to convert the input file into txt before parsing +object TXTConverterLoader extends ExternalLoader { + override def loadFromFile(file : File, p : Parsers) : Option[Paper] = { + println("parsing " + file.getPath + " using txt loader") + // looking for the format and setting the correct parameters + loadFromFile(file, p, "txt", List("-enc", "UTF-8")) + } +} + +// This object converts a file into an xml one before parsing. Must use a parser that parses XML +object XMLConverterLoader extends ExternalLoader { + def loadFromFile(file : File, p : Parsers) : Option[Paper] = { + println("parsing " + file.getPath + " using xml loader") + + // looking for the format and setting the correct parameters + SystemHelper.ext(file.getName()) match { + case "txt" => loadFromFile(file, p, "txt", List("-enc", "UTF-8")) + case "pdf" => loadFromFile(file, p, "xml", List("-xml", "-q", "-enc", "UTF-8")) + } + } +} diff --git a/src/paper/Graph.scala b/src/paper/Graph.scala index 4f92aee..716d624 100644 --- a/src/paper/Graph.scala +++ b/src/paper/Graph.scala @@ -1,31 +1,22 @@ package paper -// {"nodes":[ -// {"name":"Myriel"}, -// {"name":"Napoleon"}, -// {"name":"Mlle.Baptistine"}, -// {"name":"Mme.Magloire"}], -// -// "links":[ -// {"source":1,"target":3,"value":1}, -// {"source":2,"target":1,"value":50}, -// {"source":3,"target":2,"value":100}, -// {"source":0,"target":1,"value":1}] -// } - trait Graphs { - def getGraph(papers : List[Paper]) : Graph = { + def getGraph(paperPos:String, papers : List[Paper]) : Graph = { + println("BEGIN OF GRAPH CREATION") + val loadedPapers = if(papers == List()) CacheLoader.load(paperPos, Cache.linked) else papers // Add all papers as nodes - val nodes : List[Node] = for (p <- papers) yield makeNode(p) + val nodes : List[Node] = for (p <- loadedPapers) yield makeNode(p) // Then create all edges - val edges : List[Edge] = for (p <- papers; e <- makeEdges(p, nodes)) yield e + val edges : List[Edge] = for (p <- loadedPapers; e <- makeEdges(p, nodes)) yield e + println("END OF GRAPH CREATION") return new Graph(nodes, edges) } - def makeNode(paper : Paper) : Node = { + // MODIFICATION + def makeNode(paper : Paper) : Node = { println("Making node for " + paper.id) Node(paper.id, paper.meta("xmlpapertitle"), paper.meta("xmlauthors"), paper.meta("pdf"), paper.meta("xmldate"), paper.meta("xmlroom")) } @@ -45,14 +36,17 @@ trait Graphs { class Graph(nodes : List[Node], edges : List[Edge]) { def save : Unit = { - val f = new java.io.File("data.json") + val f = new java.io.File("graph.js") val p = new java.io.PrintWriter(f) p.println(toString) p.close } override def toString : String = { - var ret : String = "{\n" + var ret : String = "" + + // Add define + ret += "define([], function () {\n\ndata = {" // add nodes ret += "\"nodes\":" + nodes.mkString("[\n ",",\n ","\n],") + "\n" @@ -61,7 +55,7 @@ class Graph(nodes : List[Node], edges : List[Edge]) { ret += "\"links\":" + edges.mkString("[\n ",",\n ","\n]") + "\n\n" // End - ret += "}" + ret += "}\n\nreturn data;\n})" return ret } @@ -81,7 +75,7 @@ case class Node(id : Int, title : String, authors : String, pdf : String, date : } } -case class Edge(from : Int, to : Int, weight : Int) { +case class Edge(from : Int, to : Int, weight : Double) { override def toString : String = "{\"source\":" + from + ",\"target\":" + to + ",\"value\":" + weight + "}" } diff --git a/src/paper/InformationExtractors.scala b/src/paper/InformationExtractors.scala new file mode 100644 index 0000000..f5d9fd1 --- /dev/null +++ b/src/paper/InformationExtractors.scala @@ -0,0 +1,179 @@ +package paper + +trait InformationExtractor { + // This method finds the first element in a list matching a particular condition and returns it with the rest of the list + protected def findFirstOf(list: List[XMLParagraph], condition: (XMLParagraph) => Boolean): List[XMLParagraph] = list match{ + case List() => Nil + case x::xs => if(condition(x)) x::xs else findFirstOf(xs, condition) + } + + // This method find the first element matching a condition and returns the previous ones (not matching) + protected def untilFirstOf(list: List[XMLParagraph], condition: (XMLParagraph) => Boolean): List[XMLParagraph] = { + def untilFirstOf0(l: List[XMLParagraph], accu: List[XMLParagraph]): List[XMLParagraph] = l match{ + case List() => accu.reverse + case x::xs => if(condition(x)) accu.reverse else untilFirstOf0(xs, x::accu) + } + + untilFirstOf0(list, List()) + } + + // This method filters the first elements matching a condition which are adjacent + protected def filterAdjacents(list: List[XMLParagraph], condition: (XMLParagraph) => Boolean): List[XMLParagraph] = { + def filterAdjacents0(l: List[XMLParagraph], accu: List[XMLParagraph]): List[XMLParagraph] = l match { + case List() => accu + case x::xs => if(condition(x)) filterAdjacents0(xs, x::accu) + else if(accu.length == 0) filterAdjacents0(xs, accu) else accu + } + + filterAdjacents0(list, List()).reverse + } + + // This method finds the first elements matching a condition which are adjacent and returns the list of the remaining ones (after these) + protected def discardAdjacents(list: List[XMLParagraph], condition: (XMLParagraph) => Boolean): List[XMLParagraph] = { + def discardAdjacents0(l: List[XMLParagraph], accu: Int): List[XMLParagraph] = l match { + case List() => Nil + case x::xs => if(condition(x)) discardAdjacents0(xs, accu + 1) + else if(accu == 0) discardAdjacents0(xs, accu) else x::xs + } + + discardAdjacents0(list, 0) + } + + // This method takes a list of strings and returns a string containing the concatenation of all the other strings + protected def concatText(list: List[String]): String = { + def concatText0(l: List[String], accu: String): String = l match { + case List() => if(accu.length() > 0) accu.dropRight(1) else accu // Last \n replacement + case x::xs => concatText0(xs, accu + x + "\n") + } + + concatText0(list, "") + } +} + + + + +trait AuthorsExtractor1 extends InformationExtractor{ + // This method extracts the authors of the article. + // It uses the following rules: + // - They are located in the first page between title and abstract + // - Every author paragraph contains the name at the top of it, i.e. the names are located at the first line of these paragraphs + def extractAuthors(paper: Paper, xml: XMLDocument, paragraphs: List[XMLParagraph]): (List[XMLParagraph], Paper) = { + val list = untilFirstOf(paragraphs, (p: XMLParagraph) => p.hasOption(XMLParagraphOptions.JUSTIFY) && p.getPosition.getWidth >= xml.getPage(1).getPosition.getWidth / 3) + val remainingList = paragraphs.drop(list.length) + + // Finding of the most at the top paragraphs + if(list.length == 0) return (paragraphs, paper) + val minTop = list.map((p:XMLParagraph) => p.getPosition.getY).min + val unprocessedAuthorsList = list.filter((p:XMLParagraph) => p.getPosition.getY == minTop) + + // Applying the extraction processing + val authors = unprocessedAuthorsList.flatMap((p:XMLParagraph) => ("""(.+?""" + ExtractionRegexes.authorsSeparator + """)|(.+?$)""").r.findAllIn(""" [0-9]+""".r.replaceAllIn(p.getLines.head.getText, ""))) + val authorsList = authors.map((s: String) => new Author(ExtractionRegexes.authorsSeparator.r.replaceAllIn(s, ""))) + + + if(authorsList.length != 0) (remainingList, paper.setAuthors(authorsList)) + else (paragraphs, paper) + + } +} + +trait AbstractExtractor1 extends InformationExtractor{ + // This method extracts the abstract of the article. + // It uses the following rules: + // - It is the first justified paragraph + // - It is entirely contained in the first page + def extractAbstract(paper: Paper, xml: XMLDocument, paragraphs: List[XMLParagraph]): (List[XMLParagraph], Paper) = { + val list = findFirstOf(paragraphs, (p: XMLParagraph) => p.hasOption(XMLParagraphOptions.JUSTIFY) && p.getPosition.getWidth >= xml.getPage(1).getPosition.getWidth / 3) + if(list.isEmpty) return (paragraphs, paper) + val xmlFont = xml.getFontsContainer.getXMLFont(list.head.getFontID) + val abstractList = filterAdjacents(list, (p: XMLParagraph) => xmlFont.get.checkID(p.getFontID) && p.hasOption(XMLParagraphOptions.JUSTIFY)).map((p: XMLParagraph) => p.getText.replaceAll("\n", " ")) + val remainingList = discardAdjacents(list, (p: XMLParagraph) => xmlFont.get.checkID(p.getFontID) && p.hasOption(XMLParagraphOptions.JUSTIFY)) + + if(abstractList.length != 0) (remainingList, paper.setAbstract(new Abstract(concatText(abstractList)))) + else (paragraphs, paper) + } +} + +trait TitleExtractor1 extends InformationExtractor { + // This method extracts the title of the article. + // It uses the following rules: + // - The title is contained in the first page + // - It has the biggest font size + def extractTitle(paper: Paper, xml: XMLDocument, paragraphs: List[XMLParagraph]): (List[XMLParagraph], Paper) = { + // This method finds the font id having the maximum size + def findMaxSizeID(paragraphs: List[XMLParagraph]): String = { + def findMaxSizeID0(paragraphs: List[XMLParagraph], max: Int, id: String): String = paragraphs match{ + case List() => id + case x::xs => + val newSize = xml.getFontsContainer.getXMLFont(x.getFontID).get.getSize.toInt + + if(newSize > max) findMaxSizeID0(xs, newSize, xml.getFontsContainer.getXMLFont(x.getFontID).get.getID) + else findMaxSizeID0(xs, max, id) + } + + findMaxSizeID0(paragraphs, 0, "") + } + + val maxSizeIDFont = xml.getFontsContainer.getXMLFont(findMaxSizeID(paragraphs.take(10))) + val title = findFirstOf(paragraphs, p => maxSizeIDFont.get.checkID(p.getFontID) || p.hasOption(XMLParagraphOptions.PAGE_CENTERED)) + + if(title.length != 0) (title.tail, paper.setTitle(new Title(title.head.getText.replace("\n", " ")))) + else (paragraphs, paper) + } +} + + +trait BodyExtractor1 extends InformationExtractor { + // This method extracts the body of the article. + // It uses the following rules: + // - It begins with the first justified paragraph after the abstract + // - It is justified and belongs to one column + def extractBody(paper: Paper, xml: XMLDocument, paragraphs: List[XMLParagraph]): (List[XMLParagraph], Paper) = { + val firstBodyList = findFirstOf(paragraphs, (p: XMLParagraph) => p.hasOption(XMLParagraphOptions.JUSTIFY) && !p.hasOption(XMLParagraphOptions.NO_COLUMN)) + if(firstBodyList.length == 0) return (paragraphs, paper) + val bodyXmlFont = xml.getFontsContainer.getXMLFont(firstBodyList.head.getFontID) + + // Filter (applying the rules) + val bodyListWithReturn = firstBodyList.filter((p: XMLParagraph) => bodyXmlFont.get.checkID(p.getFontID) && p.hasOption(XMLParagraphOptions.JUSTIFY) && !p.hasOption(XMLParagraphOptions.NO_COLUMN)) + + // Calculating the remaining list + val lastBodyParagraph = bodyListWithReturn.last + val bodyList = bodyListWithReturn.map((p: XMLParagraph) => p.getText.replaceAll("\n", " ")) + val remainingList = findFirstOf(paragraphs, (p:XMLParagraph) => p.getText.equals(lastBodyParagraph.getText)).tail + + if(bodyList.length != 0) (remainingList, paper.setBody(new Body(concatText(bodyList)))) + else (paragraphs, paper) + } +} + +trait ReferencesExtractor1 extends InformationExtractor { + // This method extracts the references of the article. + // It uses the following rules: + // - They begin with a paragraph containing a reference title + // - Each following paragraph is a reference. + // - The method recognizes the reference format and applies a suitable extraction strategy + def extractReferences(paper: Paper, xml: XMLDocument, paragraphs: List[XMLParagraph]): (List[XMLParagraph], Paper) = { + val refParagraphs = findFirstOf(paragraphs, (p: XMLParagraph) => ("""^""" + ExtractionRegexes.referencesName + """$""").r.findFirstIn(p.getText).isDefined) + val referencesStringList = if(!refParagraphs.isEmpty) refParagraphs.tail else List() + + // This method takes a list of reference paragraphs and process them + def makeReferences(refs: List[XMLParagraph], accu: List[Reference]): List[Reference] = refs match { + case List() => accu + case x::xs => { + // Using of the good reference processor after a structure recognition + val refExtr = ReferenceProcessorExecuter.extract(x.getText.replaceAll("\n", " ")) + val title = refExtr._1 + val authors = refExtr._2 + + if(title == None || authors == None) makeReferences(xs, accu) + else makeReferences(xs, (new Reference(authors.get, title.get)::accu)) + } + } + + val refList = makeReferences(referencesStringList, List()).reverse + + if(refList.length != 0) (List(), paper.setReferences(refList)) + else (paragraphs, paper) + } +} diff --git a/src/paper/LoadPaper.scala b/src/paper/LoadPaper.scala index e7212eb..b5fca87 100644 --- a/src/paper/LoadPaper.scala +++ b/src/paper/LoadPaper.scala @@ -5,10 +5,11 @@ import scala.collection.immutable.Stream import scala.io.Source import java.io._ + object Cache { // Constants - val dir = "cache/" + val dir = "cache" + Paths.sep val parsed = "parsed" val extended = "extended" val linked = "linked" @@ -46,11 +47,11 @@ object Cache { def load(file : File) : Paper = { // Printout - println("loading file " + file.getName + " from cache") + println("Loading file " + file.getName + " from cache") // Get file and read in lines val lines : Iterator[String] = Source.fromFile(file).getLines - + // Variables var vars : Map[String, List[String]] = Map.empty.withDefaultValue(Nil) var current = "unknown"; @@ -85,6 +86,7 @@ object Cache { var m : Map[String, String] = Map.empty // Looping through all the maps for (e <- s) { + println(e.split(" -> ")(0)) m = m + Pair((e.split(" -> "))(0), (e.split(" -> "))(1)) } return m @@ -114,20 +116,17 @@ object Cache { case line => map = map + (current -> (line :: map(current))) } - //println(map.mkString("\n")) - // Now gather it val refs = for (i <- 0 to index) yield Reference(stringToAuthors(map("authors" + i)), Title(map("title" + i).head)) return refs.toList } } - - trait LoadPaper { - def load(name : String, postfix : List[String], parser : Parsers) : List[Paper] = { - + def loadAndParse(name : String, postfix : List[String], parser : Parsers, loader : FileLoader) : List[Paper] = { + println("BEGIN OF PARSING") + // Get file handle of original file or directory val orig = new File(name) @@ -135,54 +134,48 @@ trait LoadPaper { if (!orig.exists) sys.error("Something is wrong with the file or directory in the argument") // If exists, set name and file - var fnames : List[String] = List(name) - var files : List[File] = List(orig) + // In case it's a directory, let the file array contain all the files of the directory (regex utilization) + val files : List[File] = if(orig.isDirectory) SystemHelper.getFilesFromDirectory(orig) else List(orig) + val fnames : List[String] = if(orig.isDirectory) files.map(f => name ++ f.getName) else List(name) - // In case it's a directory, let the file array contain all the files of the directory - if (orig.isDirectory) { - files = orig.listFiles.filter(f => """.*\.txt$""".r.findFirstIn(f.getName).isDefined).toList - fnames = files.filter(n => n.getName != ".txt").map(f => name ++ f.getName) - } // If postfix exists, try loading from cache - var somePapers : List[Option[Paper]] = Nil - if (postfix != Nil) somePapers = files.map(f => loadFromCache(f, postfix)) + val somePapers : List[Option[Paper]] = if (postfix != Nil) files.map(f => loadFromCache(f, postfix)) else Nil // All papers that weren't loaded by cache are loaded by file - somePapers = somePapers.zip(files).map(p => if (p._1 == None) loadFromFile(p._2, parser) else p._1) + val finalPapers = somePapers.zip(files).map(p => if (p._1 == None) loadFromFile(p._2, parser, loader) else p._1) // Filter papers for None's and set index - val papers : List[Paper] = somePapers.filter(p => p != None).zipWithIndex.map({case Pair(p,i) => p.get.setIndex(i) }).toList + val papers : List[Paper] = finalPapers.filter(p => p != None).zipWithIndex.map({case Pair(p,i) => p.get.setIndex(i) }).toList + println("END OF PARSING") + return papers } - // Loads a paper from a text file and parses it - def loadFromFile(file : File, p : Parsers) : Option[Paper] = { + // Loads a paper from a text file and parses it. It has been modified in order to make loading and parsing flexible + def loadFromFile(file : File, p : Parsers, loader: FileLoader) : Option[Paper] = { - // Check if file is bad - if (checkIfBad(file)) return None - - println("parsing " + file.getPath) - val maybePaper : Option[Paper] = p.parse(Source.fromFile(file)) - // If paper exists and parsed, save it in cache - if (maybePaper != None) { - // Get index - var id = file.getPath.split('/').last.split('.').first.toInt - // Set filename and id - val paper : Paper = maybePaper.get.setMeta("file" -> file.getPath).setId(id) - // Save and return - Cache.save(paper.clean, Cache.parsed) - return Some(paper) - } + // Check if file is bad or contains non numerical values (in the name) + if (checkIfBad(file) || """[^0-9]+""".r.findFirstIn(SystemHelper.name(file.getName())).isDefined) return None + + val result = loader.loadFromFile(file, p) + // If paper doesn't exist and didn't parse, let's not parse it again - else { + if(result == None) return isBadFile(file) + else Cache.save(result.get.clean, Cache.parsed) // Save and return + + return result + } + + + def isBadFile(file: File): Option[Paper] = { + println("Couldn't parse " + file.getName()) Cache.bad(file) return None - } } - + def checkIfBad(file : File) : Boolean = { // Get file @@ -220,3 +213,21 @@ trait LoadPaper { } } } + + +object CacheLoader extends LoadPaper{ + def load(paperPos:String, postfix : String): List[Paper] = { + // Get file handle of original file or directory + val orig = new File(paperPos) + + // Check that directory or file exists + if (!orig.exists) sys.error("Problem with file path") + + val files : List[File] = if(orig.isDirectory()) orig.listFiles.toList else List(orig) + + // If postfix exists, try loading from cache + val papers : List[Option[Paper]] = files.map(f => loadFromCache(f, List(postfix))) + + return papers.filter((p:Option[Paper]) => p != None).map((p:Option[Paper]) => p.get) + } + } diff --git a/src/paper/ParsePaper.scala b/src/paper/ParsePaper.scala index 8a8b28e..7c61c6a 100644 --- a/src/paper/ParsePaper.scala +++ b/src/paper/ParsePaper.scala @@ -1,273 +1,273 @@ -package paper - -import scala.collection.immutable.Stream -import scala.collection.immutable.StringOps -import scala.util.parsing.input._ -import scala.io.Source - -abstract class Parsers { - def parse(file : Source) : Option[Paper] -} - -trait ParsePaper { - - def getText(in: Source): Stream[Char] = in.hasNext match { - case false => in.close(); Stream.Empty - case true => in.next #:: getText(in) - } - - - object Isit extends Parsers { - - def paper : Parser[Paper] = ( - //title ~ authors ~ dropLinesUntil("R EFERENCES") ~ references - dropLinesUntil("R EFERENCES") ~ references - ^^ { case b~r => Paper(0,0,Title(""),Nil,Abstract("Not saved"),Body("Not saved"),r,Map.empty,List()) } - | dropLinesUntil("References") ~ references - ^^ { case b~r => Paper(0,0,Title(""),Nil,Abstract("Not saved"),Body("Not saved"),r,Map.empty,List()) }) - - - def title : Parser[Title] = ( - line && ("Title:" ~> rest) - ^^ trim ^^ (s => Title(s.init.mkString))) - - def authors : Parser[List[Author]] = ( - line && "Author: " ~> split(", ") - ^^ (as => { - if (as.length == 0) Nil - else { - var names = as.init.map(a => a.mkString) ::: List(as.last.mkString.init) - names.map(a => Author(formatAuthor(a.mkString))) - } - })) - - def abstr : Parser[Abstract] = ( - dropLinesUntil("Abstract") ~> takeLinesUntil("I. ") - ^^ (t => Abstract(t.mkString))) - - def body : Parser[Body] = ( - takeLinesUntil("R EFERENCES") - ^^ (t => Body(t.mkString))) - - def refBracket : Parser[Input] = ( - "[" ~ rep(number) ~ "] " ^^^ Stream.Empty) - - def refLine : Parser[Input] = ( - takeLinesUntil("\n" | refBracket)) - - def refAuthors : Parser[List[Author]] = ( - until(", “") && split(", and " | " and " | ", ") - ^^ { x => x.init.map { a => Author(a.mkString) } } ) - - def refTitle : Parser[Title] = ( - until(",\"" | "\"," | "\"." | ".") ^^ (s => Title(s.mkString)) // Character modification. Possible errors in the future !!! - | success(Title(""))) - - def reference : Parser[Reference] = ( - refLine && (refAuthors ~ refTitle) - ^^ { case a~t => Reference(a, t) }) - - def references : Parser[List[Reference]] = ( - dropLinesUntil(refBracket) ~> rep(reference) ^^ cleanRefs) - - - // The function for actually parsing a paper - def parse(file : Source) : Option[Paper] = { - - paper(getText(file)) match { - case Failure(msg, rest) => println("Failure: " + msg); None - case Success(result, rest) => Some(result.setMeta("parsed" -> "yes")) - } - } - } - - - def cleanRefs(refs : List[Reference]) : List[Reference] = { - val ret = for (r <- refs if r.title.t.stripMargin != "" && r.authors.length > 0) yield { - // Clean authors - var authors = for (a <- r.authors if a.name.stripMargin.length > 2) yield Author(a.name.stripMargin) - Reference(authors, r.title) - } - return ret - } - - def formatAuthor(name : String) : String = { - var result = "" - var names = name.split(" ").filter(n => n.length > 0) - if (names.length > 0) { - result = names.init.filter(n => n.length > 0).map(n => n.head).mkString("",". ",". ") + names.last - } - return result - } - - - - type Input = Stream[Char] - sealed abstract class Result[+T] - case class ~ [+T, +U](r1 : T, r2 : U) - case class Success[T](result: T, in : Input) extends Result[T] - case class Failure(msg : String, in : Input) extends Result[Nothing] - - val lineSep : List[Char] = List('\n','\r') - val tokenSep : List[Char] = List(' ',',','.',':','[',']','-') ::: lineSep - - abstract class Parser[+T] extends (Input => Result[T]) { - p => - - def ~ [U](q: => Parser[U]) = new Parser[T~U] { - def apply(in: Input) = p(in) match { - case Success(x, in1) => q(in1) match { - case Success(y, in2) => Success(new ~(x, y), in2) - case Failure(msg, in) => Failure(msg, in) - } - case Failure(msg, in) => Failure(msg, in) - } - } - - def | [U >: T](q: => Parser[U]) = new Parser[U] { - def apply(in: Input) = p(in) match { - case s @ Success(x, rest) => s - case Failure(_,_) => q(in) - } - } - - def ^ [U](f: (T, Input) => U, g: (T, Input) => Input) : Parser[U] = new Parser[U] { - def apply(in: Input) = p(in) match { - case Success(x, rest) => Success(f(x, rest), g(x, rest)) - case Failure(msg, in) => Failure(msg, in) - } - } - - def ^^ [U](f: T => U) : Parser[U] = p ^ ({ case (x, _) => f(x) }, { case (_,r) => r }) - def ^^^ [U](v: U) = p ^^ { case _ => v } - def ~> [U](q: => Parser[U]) = (p ~ q) ^^ { case a~b => b } - def <~ [U](q: => Parser[U]) = (p ~ q) ^^ { case a~b => a } - - def &&[U](q: => Parser[U]) = new Parser[U] { - def apply(in: Input) = p(in) match { - case f @ Failure(msg, in) => f - case Success(x1, rest) => { - // Because I don't know the result of parser q, I'm backtracing to figure out what was matched - var matched = in.zipWithIndex.takeWhile{case (x,i) => !in.drop(i).equals(rest)}.unzip._1 - q(matched) match { - case Success(x2, rest2) => Success(x2, rest) - case Failure(msg, rest2) => Failure(msg, rest) - } - } - } - } - } - - //def inLine[U](q : => Parser[U]) : Parser[U] = q <~ "\n" - - def line : Parser[Input] = next(lineSep) - def token : Parser[Input] = next(tokenSep) - - def number : Parser[Input] = "1" | "2" | "3" | "4" | "5" | "6" | "7" | "8" | "9" | "0" - - def trim(s : Input) : Input = s.dropWhile(c => c==' ' || c=='\t') - def line(p : Parser[Input]) : Parser[Input] = line && p && rest - - - // Splits input by letting the parser p be the delimiter - def split(p : Parser[Input]) : Parser[List[Input]] = rep(until(p)) - - def until[U](p : => Parser[U]) : Parser[Input] = new Parser[Input] { - override def apply(in: Input) : Result[Input] = { - - def gather(s : Input, soFar : Input) : Result[Input] = p(s) match { - case Success(result, rest) => Success(soFar, rest) - case Failure(msg, Stream.Empty) => Success(soFar, Stream.Empty) - case Failure(_, _) => gather(s.drop(1), soFar ++ s.take(1)) - } - - if (in == Stream.Empty) return Failure("[Until] Reached end of input", in) - else return gather(in, Stream.Empty) - } - } - - // Much faster than the naive version, and also avoids a stackoverflow - def takeLinesUntil[U](p : => Parser[U]) : Parser[Input] = new Parser[Input] { - override def apply(in: Input) : Result[Input] = { - - def gather(s : Input, soFar : Input) : Result[Input] = { - p(s) match { - case Success(result, rest) => Success(soFar, rest) - case Failure(msg, Stream.Empty) => Success(soFar, Stream.Empty) - case Failure(_, _) => gather(s.dropWhile(c => c != '\n').drop(1), soFar ++ s.takeWhile(c => c != '\n').drop(1)) - } - } - - if (in == Stream.Empty) return Failure("[takeLinesUntil] Reached end of input", in) - else return gather(in, Stream.Empty) - } - } - - //def takeLinesUntil(p : Parser[Input]) = rep(line(p)) - def dropLinesUntil[U](p : Parser[U]) = takeLinesUntil(p) ~> success(Stream.Empty) - - - def not(s: String) : Parser[Input] = new Parser[Input] { - override def apply(in: Input) : Result[Input] = in.take(s.length).mkString == s match { - case true => Failure("Failed: " + s + " was matched", in) - case false => Success(Stream.Empty, in) - } - } - - def opt [T](q : => Parser[T]) : Parser[Option[T]] = ( - q ^^ { case a => Some(a) } - | success(None)) - - def rep [T](p : => Parser[T]) : Parser[List[T]] = ( - p ~ rep(p) ^^ { case a~b => a::b } - | success(List())) - - def repsep [T, U](p : => Parser[T], sep : => Parser[U]) : Parser[List[T]] = ( - p ~ rep(sep ~> p) ^^ { case a~b => a::b } - | success(List())) - - def success[T](v: T) : Parser[T] = new Parser[T] { - override def apply(in: Input) : Result[T] = Success(v,in) - } - - def rest : Parser[Input] = new Parser[Input] { - override def apply(in: Input) : Result[Input] = { - in.length match { - case 0 => Failure("[Rest] Reached end of input",in) - case n => Success(in, Stream.Empty) - } - } - } - - def next(sep : List[Char]) : Parser[Input] = new Parser[Input] { - def apply(in: Input) = in.span(c => !sep.contains(c)) match { - case (Stream.Empty, Stream.Empty) => Failure("[Next] Reached end of input", in) - case (head, rest) => Success(head.init, rest.drop(1)) - } - } - - implicit def predicate(p: String => Boolean) : Parser[Input] = new Parser[Input] { - def apply(in: Input) = { - p(in.mkString) match { - case true => Success(Stream.Empty, in) - case false => Failure("Couldn't match predicate on:\n" + in.mkString, Stream.Empty) - } - } - } - - implicit def str(t: String) : Parser[Input] = new Parser[Stream[Char]] { - def apply(in: Input) = in.take(t.length).mkString == t match { - case true => Success(in.take(t.length), in.drop(t.length)) - case _ => Failure("Couldn't match expected: '" + t + "' given " + in.mkString, in) - } - } - - implicit def resultToStream(r: Result[Input]) : Input = r match { - case Success(x, rest) => x - case Failure(_,_) => Stream.Empty - } - - // //implicit def Stream2Str(s : Stream[Char]) : String = s.mkString - -} - +package paper + +import scala.collection.immutable.Stream +import scala.collection.immutable.StringOps +import scala.util.parsing.input._ +import scala.io.Source +import java.io.File + +abstract class Parsers { + def parse(file : Source) : Option[Paper] +} + +trait ParsePaper { + + def getText(in: Source): Stream[Char] = in.hasNext match { + case false => in.close(); Stream.Empty + case true => in.next #:: getText(in) + } + + + object Isit extends Parsers { + + def paper : Parser[Paper] = ( + //title ~ authors ~ dropLinesUntil("R EFERENCES") ~ references + dropLinesUntil("R EFERENCES") ~ references + ^^ { case b~r => Paper(0,0,Title(""),Nil,Abstract("Not saved"),Body("Not saved"),r,Map.empty,List()) } + | dropLinesUntil("References") ~ references + ^^ { case b~r => Paper(0,0,Title(""),Nil,Abstract("Not saved"),Body("Not saved"),r,Map.empty,List()) }) + + + def title : Parser[Title] = ( + line && ("Title:" ~> rest) + ^^ trim ^^ (s => Title(s.init.mkString))) + + def authors : Parser[List[Author]] = ( + line && "Author: " ~> split(", ") + ^^ (as => { + if (as.length == 0) Nil + else { + var names = as.init.map(a => a.mkString) ::: List(as.last.mkString.init) + names.map(a => Author(formatAuthor(a.mkString))) + } + })) + + def abstr : Parser[Abstract] = ( + dropLinesUntil("Abstract") ~> takeLinesUntil("I. ") + ^^ (t => Abstract(t.mkString))) + + def body : Parser[Body] = ( + takeLinesUntil("R EFERENCES") + ^^ (t => Body(t.mkString))) + + def refBracket : Parser[Input] = ( + "[" ~ rep(number) ~ "] " ^^^ Stream.Empty) + + def refLine : Parser[Input] = ( + takeLinesUntil("\n" | refBracket)) + + def refAuthors : Parser[List[Author]] = ( + until(", “") && split(", and " | " and " | ", ") + ^^ { x => x.init.map { a => Author(a.mkString) } } ) + + def refTitle : Parser[Title] = ( + until(",\"" | "\"," | "\"." | ".") ^^ (s => Title(s.mkString)) // Character modification. Possible errors in the future !!! + | success(Title(""))) + + def reference : Parser[Reference] = ( + refLine && (refAuthors ~ refTitle) + ^^ { case a~t => Reference(a, t) }) + + def references : Parser[List[Reference]] = ( + dropLinesUntil(refBracket) ~> rep(reference) ^^ cleanRefs) + + + // The function for actually parsing a paper + def parse(file : Source) : Option[Paper] = { + paper(getText(file)) match { + case Failure(msg, rest) => println("Failure: " + msg); None + case Success(result, rest) => Some(result.setMeta("parsed" -> "yes")) + } + } + } + + + def cleanRefs(refs : List[Reference]) : List[Reference] = { + val ret = for (r <- refs if r.title.t.stripMargin != "" && r.authors.length > 0) yield { + // Clean authors + var authors = for (a <- r.authors if a.name.stripMargin.length > 2) yield Author(a.name.stripMargin) + Reference(authors, r.title) + } + return ret + } + + def formatAuthor(name : String) : String = { + var result = "" + var names = name.split(" ").filter(n => n.length > 0) + if (names.length > 0) { + result = names.init.filter(n => n.length > 0).map(n => n.head).mkString("",". ",". ") + names.last + } + return result + } + + + + type Input = Stream[Char] + sealed abstract class Result[+T] + case class ~ [+T, +U](r1 : T, r2 : U) + case class Success[T](result: T, in : Input) extends Result[T] + case class Failure(msg : String, in : Input) extends Result[Nothing] + + val lineSep : List[Char] = List('\n','\r') + val tokenSep : List[Char] = List(' ',',','.',':','[',']','-') ::: lineSep + + abstract class Parser[+T] extends (Input => Result[T]) { + p => + + def ~ [U](q: => Parser[U]) = new Parser[T~U] { + def apply(in: Input) = p(in) match { + case Success(x, in1) => q(in1) match { + case Success(y, in2) => Success(new ~(x, y), in2) + case Failure(msg, in) => Failure(msg, in) + } + case Failure(msg, in) => Failure(msg, in) + } + } + + def | [U >: T](q: => Parser[U]) = new Parser[U] { + def apply(in: Input) = p(in) match { + case s @ Success(x, rest) => s + case Failure(_,_) => q(in) + } + } + + def ^ [U](f: (T, Input) => U, g: (T, Input) => Input) : Parser[U] = new Parser[U] { + def apply(in: Input) = p(in) match { + case Success(x, rest) => Success(f(x, rest), g(x, rest)) + case Failure(msg, in) => Failure(msg, in) + } + } + + def ^^ [U](f: T => U) : Parser[U] = p ^ ({ case (x, _) => f(x) }, { case (_,r) => r }) + def ^^^ [U](v: U) = p ^^ { case _ => v } + def ~> [U](q: => Parser[U]) = (p ~ q) ^^ { case a~b => b } + def <~ [U](q: => Parser[U]) = (p ~ q) ^^ { case a~b => a } + + def &&[U](q: => Parser[U]) = new Parser[U] { + def apply(in: Input) = p(in) match { + case f @ Failure(msg, in) => f + case Success(x1, rest) => { + // Because I don't know the result of parser q, I'm backtracing to figure out what was matched + var matched = in.zipWithIndex.takeWhile{case (x,i) => !in.drop(i).equals(rest)}.unzip._1 + q(matched) match { + case Success(x2, rest2) => Success(x2, rest) + case Failure(msg, rest2) => Failure(msg, rest) + } + } + } + } + } + + //def inLine[U](q : => Parser[U]) : Parser[U] = q <~ "\n" + + def line : Parser[Input] = next(lineSep) + def token : Parser[Input] = next(tokenSep) + + def number : Parser[Input] = "1" | "2" | "3" | "4" | "5" | "6" | "7" | "8" | "9" | "0" + + def trim(s : Input) : Input = s.dropWhile(c => c==' ' || c=='\t') + def line(p : Parser[Input]) : Parser[Input] = line && p && rest + + + // Splits input by letting the parser p be the delimiter + def split(p : Parser[Input]) : Parser[List[Input]] = rep(until(p)) + + def until[U](p : => Parser[U]) : Parser[Input] = new Parser[Input] { + override def apply(in: Input) : Result[Input] = { + + def gather(s : Input, soFar : Input) : Result[Input] = p(s) match { + case Success(result, rest) => Success(soFar, rest) + case Failure(msg, Stream.Empty) => Success(soFar, Stream.Empty) + case Failure(_, _) => gather(s.drop(1), soFar ++ s.take(1)) + } + + if (in == Stream.Empty) return Failure("[Until] Reached end of input", in) + else return gather(in, Stream.Empty) + } + } + + // Much faster than the naive version, and also avoids a stackoverflow + def takeLinesUntil[U](p : => Parser[U]) : Parser[Input] = new Parser[Input] { + override def apply(in: Input) : Result[Input] = { + + def gather(s : Input, soFar : Input) : Result[Input] = { + p(s) match { + case Success(result, rest) => Success(soFar, rest) + case Failure(msg, Stream.Empty) => Success(soFar, Stream.Empty) + case Failure(_, _) => gather(s.dropWhile(c => c != '\n').drop(1), soFar ++ s.takeWhile(c => c != '\n').drop(1)) + } + } + + if (in == Stream.Empty) return Failure("[takeLinesUntil] Reached end of input", in) + else return gather(in, Stream.Empty) + } + } + + //def takeLinesUntil(p : Parser[Input]) = rep(line(p)) + def dropLinesUntil[U](p : Parser[U]) = takeLinesUntil(p) ~> success(Stream.Empty) + + + def not(s: String) : Parser[Input] = new Parser[Input] { + override def apply(in: Input) : Result[Input] = in.take(s.length).mkString == s match { + case true => Failure("Failed: " + s + " was matched", in) + case false => Success(Stream.Empty, in) + } + } + + def opt [T](q : => Parser[T]) : Parser[Option[T]] = ( + q ^^ { case a => Some(a) } + | success(None)) + + def rep [T](p : => Parser[T]) : Parser[List[T]] = ( + p ~ rep(p) ^^ { case a~b => a::b } + | success(List())) + + def repsep [T, U](p : => Parser[T], sep : => Parser[U]) : Parser[List[T]] = ( + p ~ rep(sep ~> p) ^^ { case a~b => a::b } + | success(List())) + + def success[T](v: T) : Parser[T] = new Parser[T] { + override def apply(in: Input) : Result[T] = Success(v,in) + } + + def rest : Parser[Input] = new Parser[Input] { + override def apply(in: Input) : Result[Input] = { + in.length match { + case 0 => Failure("[Rest] Reached end of input",in) + case n => Success(in, Stream.Empty) + } + } + } + + def next(sep : List[Char]) : Parser[Input] = new Parser[Input] { + def apply(in: Input) = in.span(c => !sep.contains(c)) match { + case (Stream.Empty, Stream.Empty) => Failure("[Next] Reached end of input", in) + case (head, rest) => Success(head.init, rest.drop(1)) + } + } + + implicit def predicate(p: String => Boolean) : Parser[Input] = new Parser[Input] { + def apply(in: Input) = { + p(in.mkString) match { + case true => Success(Stream.Empty, in) + case false => Failure("Couldn't match predicate on:\n" + in.mkString, Stream.Empty) + } + } + } + + implicit def str(t: String) : Parser[Input] = new Parser[Stream[Char]] { + def apply(in: Input) = in.take(t.length).mkString == t match { + case true => Success(in.take(t.length), in.drop(t.length)) + case _ => Failure("Couldn't match expected: '" + t + "' given " + in.mkString, in) + } + } + + implicit def resultToStream(r: Result[Input]) : Input = r match { + case Success(x, rest) => x + case Failure(_,_) => Stream.Empty + } + + // //implicit def Stream2Str(s : Stream[Char]) : String = s.mkString + +} + diff --git a/src/paper/RecognitionClasses.scala b/src/paper/RecognitionClasses.scala new file mode 100644 index 0000000..0a5f49b --- /dev/null +++ b/src/paper/RecognitionClasses.scala @@ -0,0 +1,124 @@ +package paper + +object ExtractionRegexes { + val and = """(and|&)""" + val quoteB = """(“|\")""" + val quoteE = """(”|\")""" + val tabulation = """ \t """ + val authorsSeparator = """( ?, and | ?, | and | ?, \t | \t | & )""" + val referencesName = """(REFERENCES|R EFERENCES|References|R eferences)""" + val titleTermination = """[\.\?]""" +} + +// This class is a the top of the hierarchy +abstract class ReferenceProcessor { + def extract(ref: String): (Option[Title], Option[List[Author]]) +} + +// This class tries to extract information using a list of existent defined formats +object ReferenceProcessorExecuter { + val recognitionClasses: List[ReferenceProcessor] = List[ReferenceProcessor](NumberedReferenceProcessor1, NumberedReferenceProcessor2, TextualReferenceProcessor1) + + def extract(ref: String): (Option[Title], Option[List[Author]]) = { + def extract0(refProcessors: List[ReferenceProcessor]): (Option[Title], Option[List[Author]]) = refProcessors match{ + case List() => (None, None) + case x::List() => x.extract(ref) + case x::xs => { + val input = extract0(xs) + val title = input._1 + val authors = input._2 + + if(title == None && authors == None) x.extract(ref) + else (title, authors) + } + } + + extract0(recognitionClasses.reverse) + } +} + + + +// This class can extract information out of a reference having the following format: [digit] authors "title" +object NumberedReferenceProcessor1 extends ReferenceProcessor { + def extract(ref: String): (Option[Title], Option[List[Author]]) = { + def extractTitle: Option[Title] = { + val t = (ExtractionRegexes.quoteB + """.+""" + ExtractionRegexes.quoteE).r.findFirstIn(ref) + if(!t.isDefined) return None + + // Dropping the " characters + val title = t.get.drop(1).dropRight(1) + val finalTitle = if(title.last.equals(',')) title.dropRight(1) else title + Some(new Title(finalTitle)) + } + + def extractAuthors: Option[List[Author]] = { + val t = ("""^\[\d+\].+""" + ExtractionRegexes.quoteB).r.findFirstIn(ref) + if(!t.isDefined) return None + + val totAuths = t.get + val auths = ("""^\[\d+\]""").r.replaceAllIn(totAuths.dropRight(1), "") + + val authorsStringList = ("""([A-Z]\.[ -])+.+?( """ + ExtractionRegexes.and + """|,)""").r.findAllIn(auths).toList + + Some(authorsStringList.map((s:String) => new Author(("""( """ + ExtractionRegexes.and + """)""").r.replaceAllIn(s, "").replace(",", "")))) + } + + if(("""^\[\d+\].+""" + ExtractionRegexes.quoteB + """.+""" + ExtractionRegexes.quoteE + """(.+)?$""").r.findFirstIn(ref).isDefined){ + (extractTitle, extractAuthors) + } else (None, None) + } +} + + +// This class can extract information out of a reference having the following format: [digit] authors. title +object NumberedReferenceProcessor2 extends ReferenceProcessor { + def extract(ref: String): (Option[Title], Option[List[Author]]) = { + def extract0(ref: String): (Option[Title], Option[List[Author]]) = { + val authRegex = ("""([A-Z]\.[ -])+.+?( """ + ExtractionRegexes.and + """|,|, """ + ExtractionRegexes.and + """|\.)""").r + + val ref2 = ("""^\[\d+\]( )""").r.replaceAllIn(ref, "") + val authorsStringList = authRegex.findAllIn(ref2).toList + + val auths = Some(authorsStringList.map((s:String) => new Author(("""( """ + ExtractionRegexes.and + """)""").r.replaceAllIn(s, "").replace(",", "").dropRight(1)))) + + val t = ("""^.+?""" + ExtractionRegexes.titleTermination + """""").r.findFirstIn(authRegex.replaceAllIn(ref2, "")) + if(!t.isDefined) return (None, None) + + // Dropping until a the capital character is founded + val title = Some(new Title(t.get.replace(".", "").dropWhile((c:Char) => """[^A-Z]""".r.findFirstIn(""+c).isDefined))) + + (title, auths) + } + + if(("""^\[\d+\].+$""").r.findFirstIn(ref).isDefined) extract0(ref) + else (None, None) + } +} + +// This class can extract information out of a reference having the following format: authors (digits) title +object TextualReferenceProcessor1 extends ReferenceProcessor { + def extract(ref: String): (Option[Title], Option[List[Author]]) = { + def extractTitle: Option[Title] = { + val t = ("""\)\..+?""" + ExtractionRegexes.titleTermination + """""").r.findFirstIn(ref) + if(!t.isDefined) return None + + Some(new Title(t.get.drop(3))) + } + + def extractAuthors: Option[List[Author]] = { + val t = """^.+?\(""".r.findFirstIn(ref) + if(!t.isDefined) return None + + // Dropping the " (" string + val auths = t.get.dropRight(2) + + val authorsStringList = (""".+?,( [A-Z]\.)+(, (""" + ExtractionRegexes.and + """ )?)?""").r.findAllIn(auths).toList + + Some(authorsStringList.map((s:String) => new Author(("""( """ + ExtractionRegexes.and + """)""").r.replaceAllIn(s, "").replace(",", "")))) + } + + if(("""^.+?\(\d+\)\..+?\..+?\.$""").r.findFirstIn(ref).isDefined) (extractTitle, extractAuthors) + else (None, None) + } +} diff --git a/src/paper/Terms.scala b/src/paper/Terms.scala index 5dec6a8..af0cc46 100644 --- a/src/paper/Terms.scala +++ b/src/paper/Terms.scala @@ -1,79 +1,106 @@ -package paper - -/** Abstract Syntax Trees for terms. */ -sealed abstract class Term - - -// Try to keep this immutable -case class Paper(val id : Int, - val index : Int, - val title: Title, - val authors: List[Author], - val abstr: Abstract, - val body: Body, - val refs: List[Reference], - val meta: Map[String, String], - val links : List[Link]) extends Term { - - // Add a field that contains options, such as parsed and linked - - val parsed : Boolean = false - val linked : Boolean = false - - override def toString: String = title + "\n" + authors.mkString(", ") + "\n" + abstr + "\n" + body + "\n" + refs.mkString("\n") - - def getDistinctNames : List[String] = { - val as = authors ::: refs.flatMap(r => r.authors) - val names = as.map(a => a.toString) - return names.distinct - } - - def clean : Paper = - return Paper(id, index, title, authors.filter(a => a.name.length > 4), abstr, body, refs.map(r => r.clean), meta, links) - - def setMeta(p : (String, String)) : Paper = - return Paper(id, index, title, authors, abstr, body, refs, meta + p, links) - - def setTitle(t : String) : Paper = - return Paper(id, index, Title(t), authors, abstr, body, refs, meta, links) - - def setAuthors(as : List[Author]) : Paper = - return Paper(id, index, title, as, abstr, body, refs, meta, links) - - def hasMeta(l : String) : Boolean = (meta.get(l) == None) - - def setId(newId : Int) : Paper = - return Paper(newId, index, title, authors, abstr, body, refs, meta, links) - - def setIndex(newIndex : Int) : Paper = - return Paper(id, newIndex, title, authors, abstr, body, refs, meta, links) - - def setLinks(newLinks : List[Link]) : Paper = - return Paper(id, index, title, authors, abstr, body, refs, meta, newLinks) -} - -case class Title(t: String) extends Term { - override def toString: String = t -} - -case class Author(name: String) extends Term { - override def toString: String = name -} - -case class Abstract(text: String) extends Term { - override def toString : String = "Abstract:\t" + text.take(40) + " ... " -} - -case class Body(text: String) extends Term { - override def toString: String = "Body:\t\t" + text.take(100) ++ " ... \n" -} - -case class Reference(authors: List[Author], title: Title) extends Term { - def clean : Reference = return Reference(authors.filter(a => a.name.stripMargin.length > 0), title) - override def toString : String = authors.mkString("\n") + "\n--\n" + title -} - -case class Link(index : Int, weight : Int) extends Term { - override def toString : String = index + " " + weight -} - +package paper + +/** Abstract Syntax Trees for terms. */ +sealed abstract class Term + + +// Try to keep this immutable +case class Paper(val id : Int, + val index : Int, + val title: Title, + val authors: List[Author], + val abstr: Abstract, + val body: Body, + val refs: List[Reference], + val meta: Map[String, String], + val links : List[Link]) extends Term { + + // Add a field that contains options, such as parsed and linked + + val parsed : Boolean = false + val linked : Boolean = false + + override def toString: String = title + "\n" + authors.mkString(", ") + "\n" + abstr + "\n" + body + "\n" + refs.mkString("\n") + + def getDistinctNames : List[String] = { + val as = authors ::: refs.flatMap(r => r.authors) + val names = as.map(a => a.toString) + return names.distinct + } + + def getTitle : Title = title + + def getAuthors : List[Author] = authors + def getAbstract : Abstract = abstr + def getBody : Body = body + def getReferences : List[Reference] = refs + + def clean : Paper = + return Paper(id, index, title, authors.filter(a => a.name.length > 4), abstr, body, refs.map(r => r.clean), meta, links) + + def setMeta(p : (String, String)) : Paper = + return Paper(id, index, title, authors, abstr, body, refs, meta + p, links) + + def setTitle(t : Title) : Paper = + return Paper(id, index, t, authors, abstr, body, refs, meta, links) + + def setAuthors(as : List[Author]) : Paper = + return Paper(id, index, title, as, abstr, body, refs, meta, links) + + def hasMeta(l : String) : Boolean = (meta.get(l) != None) + + def setId(newId : Int) : Paper = + return Paper(newId, index, title, authors, abstr, body, refs, meta, links) + + def setIndex(newIndex : Int) : Paper = + return Paper(id, newIndex, title, authors, abstr, body, refs, meta, links) + + def setLinks(newLinks : List[Link]) : Paper = + return Paper(id, index, title, authors, abstr, body, refs, meta, newLinks) + + def setAbstract(newAbstract : Abstract) : Paper = + return Paper(id, index, title, authors, newAbstract, body, refs, meta, links) + + def setBody(newBody : Body) : Paper = + return Paper(id, index, title, authors, abstr, newBody, refs, meta, links) + + def setReferences(newRefs : List[Reference]) : Paper = + return Paper(id, index, title, authors, abstr, body, newRefs, meta, links) +} + +case class Title(t: String) extends Term { + override def toString: String = t + + def getText: String = t +} + +case class Author(name: String) extends Term { + override def toString: String = name + + def getName: String = name +} + +case class Abstract(text: String) extends Term { + override def toString : String = "Abstract:\t" + text.take(40) + " ... " + + def getText: String = text +} + +case class Body(text: String) extends Term { + override def toString: String = "Body:\t\t" + text.take(100) ++ " ... \n" + + def getText: String = text +} + +case class Reference(authors: List[Author], title: Title) extends Term { + def clean : Reference = return Reference(authors.filter(a => a.name.stripMargin.length > 0), title) + override def toString : String = authors.mkString("\n") + "\n--\n" + title + + def getAuthors: List[Author] = authors + def getTitle: Title = title +} + +case class Link(index : Int, weight : Int) extends Term { + override def toString : String = index + " " + weight +} + diff --git a/src/paper/XMLObjects.scala b/src/paper/XMLObjects.scala new file mode 100644 index 0000000..66c8142 --- /dev/null +++ b/src/paper/XMLObjects.scala @@ -0,0 +1,106 @@ +package paper + +// This class represents a position (can be a paragraph or even a page) +class XMLPosition(x: Int, y: Int, width: Int, height: Int) { + def getX: Int = x + def getY: Int = y + def getWidth: Int = width + def getHeight: Int = height + + override def toString(): String = "[x= " + x + ", y= " + y + ", width= " + width + ", height= " + height + "]" +} + +// This class defines a particular font (with the totality of xml ids defining it) +class XMLFont(IDs:String, size: String, family: String, color: String) { + def getID: String = IDs + def getSize: String = size + def getFamily: String = family + def getColor: String = color + + // This method adds a new xml id to the definition of the font + def addID(id: String): XMLFont = new XMLFont(IDs + "-" + id, size, family, color) + + // This method checks if a particular xml id is defining that font. Use "xmlFont.checkID(myid)" instead of "xmlFont.getID == myid" + def checkID(id: String): Boolean = ("(^(" + IDs.replace("-", "|") + ")$)|(^(" + IDs.replace("-", "|") + ")-)|(-(" + IDs.replace("-", "|") + ")$)|(-(" + IDs.replace("-", "|") + ")-)").r.findFirstIn(id).isDefined + + // This method checks if two xml font are basically the same + def compareXMLFont(font: XMLFont): Boolean = (size == font.getSize && family == font.getFamily && color == font.getColor) +} + +// This class contains the fonts of the document +class XMLFontsContainer(fonts: List[XMLFont]){ + def getXMLFont(id: String): Option[XMLFont] = fonts.find(f => f.checkID(id)) + def filter (f:XMLFont => Boolean) = fonts.filter(f) +} + + +object XMLParagraphOptions { + val CENTERED = "CTR" + val PAGE_CENTERED = "PCT" + val JUSTIFY = "JFY" + val COLUMN_LEFT = "CLL" + val COLUMN_RIGHT = "CLR" + val NO_COLUMN = "NCL" + val ENUMERATION = "ENM" + val NONE = "NON" +} + +class XMLParagraphOptionsContainer(value: String) { + def getValue: String = value + def addOption(option: String): XMLParagraphOptionsContainer = if(!hasOption(option)) new XMLParagraphOptionsContainer(value + "|" + option) else this + def hasOption(option: String): Boolean = ("^(" + value + ")$").r.findFirstIn(option).isDefined + def removeOption(option: String): XMLParagraphOptionsContainer = new XMLParagraphOptionsContainer(value.replace("|" + option, "")) +} + +class XMLLine(fontID: String, position: XMLPosition, text: String) { + def getFontID: String = fontID + def getPosition: XMLPosition = position + def getText: String = text + + def setFontID(newFont: String) = new XMLLine(newFont, position, text) + def addText(newLine: XMLLine): XMLLine = new XMLLine(fontID, new XMLPosition(position.getX, position.getY, (newLine.getPosition.getX + newLine.getPosition.getWidth) - position.getX, position.getHeight), text + newLine.getText) + def setText(newText: String): XMLLine = new XMLLine(fontID, position, newText) +} + +// This class represents a paragraph contained in a page +class XMLParagraph(fontID: String, position: XMLPosition, options: XMLParagraphOptionsContainer, lines: List[XMLLine], linesSeparator: String, text: String, enumFormat: String) { + def getFontID: String = fontID + def getPosition: XMLPosition = position + def getText: String = text + def getLines: List[XMLLine] = lines + def getEnumerationFormat: String = enumFormat + + def addOption(option: String): XMLParagraph = new XMLParagraph(fontID, position, options.addOption(option), lines, linesSeparator, text, enumFormat) + def hasOption(option: String): Boolean = options.hasOption(option) + def removeOption(option: String): XMLParagraph = new XMLParagraph(fontID, position, options.removeOption(option), lines, linesSeparator, text, enumFormat) + def getOptionsValue: String = options.getValue + + // This method adds a new XMLLine to the paragraph. However, the added line will be the first one on the list, so pay attention! + // The text and position arguments will be correctly updated + def addLine(line: XMLLine): XMLParagraph = new XMLParagraph(fontID, new XMLPosition(scala.math.min(position.getX, line.getPosition.getX), position.getY, scala.math.max(position.getX + position.getWidth, line.getPosition.getX + line.getPosition.getWidth) - scala.math.min(position.getX, line.getPosition.getX), (line.getPosition.getY + line.getPosition.getHeight) - position.getY), options, line :: lines, linesSeparator, text + linesSeparator + line.getText, enumFormat) + def addParagraph(newParagraph: XMLParagraph): XMLParagraph = new XMLParagraph(fontID, new XMLPosition(scala.math.min(position.getX, newParagraph.getPosition.getX), position.getY, scala.math.max(position.getX + position.getWidth, newParagraph.getPosition.getX + newParagraph.getPosition.getWidth) - scala.math.min(position.getX, newParagraph.getPosition.getX), (newParagraph.getPosition.getY + newParagraph.getPosition.getHeight) - position.getY), options, newParagraph.getLines ::: lines, linesSeparator, text + linesSeparator + newParagraph.getText, enumFormat) + def reverseLines: XMLParagraph = new XMLParagraph(fontID, position, options, lines.reverse, linesSeparator, text, enumFormat) +} + +// This class is the representation of a page +class XMLPage(number: Int, position: XMLPosition, paragraphs: List[XMLParagraph]) { + def getNumber: Int = number + def getPosition: XMLPosition = position + def getParagraphs: List[XMLParagraph] = paragraphs +} + +// This class defines the entire xml file structure +class XMLDocument(fontsContainer: XMLFontsContainer, pages: List[XMLPage]) { + private val paragraphs = { + def get(pageList: List[XMLPage], accu: List[XMLParagraph]):List[XMLParagraph] = pageList match { + case List() => accu + case x::xs => get(xs, accu ::: x.getParagraphs) + } + + get(pages, List()) + } + + def getFontsContainer: XMLFontsContainer = fontsContainer + def getPage(pageNumber: Int): XMLPage = (pages.filter(p => p.getNumber == pageNumber)).head + def getParagraphs: List[XMLParagraph] = paragraphs +} diff --git a/src/paper/XMLObjectsManager.scala b/src/paper/XMLObjectsManager.scala new file mode 100644 index 0000000..c5893e0 --- /dev/null +++ b/src/paper/XMLObjectsManager.scala @@ -0,0 +1,115 @@ +package paper +import scala.xml.XML +import scala.xml.Elem +import scala.xml.NodeSeq + + +object XMLObjectsManager { + + def getCleanXMLParagraph(lineSeparator: String): XMLParagraph = new XMLParagraph("", new XMLPosition(0, 0, 0, 0), new XMLParagraphOptionsContainer(XMLParagraphOptions.NONE), List(), lineSeparator, "", "") + + // Since the xml file sometimes creates different fonts with the same features (size, family, etc), it + // is important to create a structure that takes into account this detail and hides it. + // The XMLFont class performs this job. XMLFontsContainer just contains the list of XMLFont objects + private def getFontsContainer(xml: Elem): Option[XMLFontsContainer] = { + val pages = (xml \\ "page") + + + // This method updates the previous XMLFont list with the new fonts defined in the page. + def getXMLFontListFromPage(previousList: List[XMLFont], pageNumber: String): List[XMLFont] = { + // Recursive method that runs through the page's fonts list and updates the given XMLFont list + def constructFontLists(fonts: NodeSeq, accu: List[XMLFont]): List[XMLFont] = { + + // This method runs through the given XMLFont list and checks if the new XMLFont defined object is already in the list + def checkIfExist(xmlFont: XMLFont, list: List[XMLFont], accu: List[XMLFont]): List[XMLFont] = list match{ + case List() => (xmlFont :: accu).reverse // Putting the new XMLFont at the end of the list + case x::xs => if(x.compareXMLFont(xmlFont)) (x.addID(xmlFont.getID) :: accu).reverse ::: xs // Updating the given XMLFont list + else checkIfExist(xmlFont, xs, x :: accu) + } + + fonts.isEmpty match { + case true => accu + case false => { + // Creating then new XMLFont object according to the font parameters + val xmlFont = new XMLFont((fonts.head \ "@id").text, (fonts.head \ "@size").text, (fonts.head \ "@family").text, (fonts.head \ "@color").text) + constructFontLists(fonts.tail, checkIfExist(xmlFont, accu, List())) + } + } + + } + + // Getting the page + val page = pages filter((n) => (n \ "@number").text.equals(pageNumber)) + + if(page.length != 1) previousList + else { + val fonts = page.head \\ "fontspec" + + // Updating the XMLFont list + constructFontLists(fonts, previousList) + } + } + + // Method for running through the pages list + def constructUntilPage(pagesNumber: Int, accu: List[XMLFont], currentPage: Int): List[XMLFont] = { + if(currentPage > pagesNumber) accu + else constructUntilPage(pagesNumber, getXMLFontListFromPage(accu, currentPage.toString()), currentPage + 1) + } + + // Getting the xml fonts list + val xmlFonts = constructUntilPage(pages.length, List(), 1) + + // Creating the container + if(xmlFonts.length >= 1) Some(new XMLFontsContainer(xmlFonts)) + else None + } + + // This method creates the pages of the document. The final list is reversed + private def constructXMLPages(pages: NodeSeq, fontsContainer: Option[XMLFontsContainer], lineSeparator: String): Option[List[XMLPage]] = { + // This method constructs the xml page + def constructXMLPage(page: xml.Node): Option[XMLPage] = { + try { + // First we get the global page parameters + val number = (page \ "@number").text.toInt + val x = (page \ "@top").text.toInt + val y = (page \ "@left").text.toInt + val width = (page \ "@width").text.toInt + val height = (page \ "@height").text.toInt + val position = new XMLPosition(x, y, width, height) + // Construction of the paragraphs + val paragraphs = XMLParagraphsConstructor.constructXMLParagraphs(page, position, fontsContainer.get, lineSeparator) + + if (paragraphs != None) Some(new XMLPage(number, position, paragraphs.get)) + else None + }catch { + case _ => None + } + + } + + // Recursive method in order to run through the pages list + def constructXMLPages0(pages: NodeSeq, accu: Option[List[XMLPage]]): Option[List[XMLPage]] = { + if(pages.isEmpty) accu + else { + val page = constructXMLPage(pages.head) + + if(page == None) None + else constructXMLPages0(pages.tail, Some(page.get :: accu.get)) + } + } + + if(fontsContainer != None) constructXMLPages0(pages, Some(List())) + else None + } + + // This method constructs the xml document of the file + def constructXMLDocument(xml:Elem, lineSeparator:String): Option[XMLDocument] = { + // getting fontsContainer and xml pages + val fontsContainer = getFontsContainer(xml) + val xmlPages = constructXMLPages(xml \\ "page", fontsContainer, lineSeparator) + + // If everything is fine, then create the document + if(xmlPages != None) Some(new XMLDocument(fontsContainer.get, xmlPages.get.reverse)) + else None + } +} diff --git a/src/paper/XMLParagraphsConstructor.scala b/src/paper/XMLParagraphsConstructor.scala new file mode 100644 index 0000000..cd1d9c3 --- /dev/null +++ b/src/paper/XMLParagraphsConstructor.scala @@ -0,0 +1,180 @@ +package paper +import scala.xml.NodeSeq + +object XMLParagraphsConstructor { + private class ParagraphDelimiter(value: Int, tolerance: Int) { + // !!! Equal to = (assignment, not comparing) + def == (cValue: Int): ParagraphDelimiter = new ParagraphDelimiter(cValue, tolerance) + + def >= (cValue: Int): Boolean = (cValue <= value + tolerance) + def <= (cValue: Int): Boolean = (cValue >= value - tolerance) + def > (cValue: Int): Boolean = (cValue < value + tolerance) + def < (cValue: Int): Boolean = (cValue > value - tolerance) + + // Comparing + def === (cValue: Int): Boolean = this >= cValue && this <= cValue + def != (cValue: Int): Boolean = !(this === cValue) + } + + + def constructXMLParagraphs(page: xml.Node, pagePosition: XMLPosition, fontsContainer: XMLFontsContainer, lineSeparator: String): Option[List[XMLParagraph]] = { + val lines = page \\ "text" + val pageCenter = new ParagraphDelimiter(pagePosition.getWidth / 2, 3) + + def createXMLLine(line: xml.Node) = new XMLLine((line \ "@font").text, new XMLPosition((line \ "@left").text.toInt, (line \ "@top").text.toInt, (line \ "@width").text.toInt, (line \ "@height").text.toInt), line.text) + + // This method incorporates the lines and finds out the real line font (see documentation) + def incorporate(nextLines: NodeSeq, currentLine: XMLLine, fontLengthMap: Map[String, Int]): (NodeSeq, XMLLine) = nextLines.isEmpty match{ + case true => { + // Determining of the real line font + val maxFontLength = fontLengthMap.toList.map((f:(String, Int)) => f._2).max + (nextLines, currentLine.setFontID(fontLengthMap.filter((p:(String, Int)) => p._2 == maxFontLength).head._1)) + } + case false => + val nextLine = createXMLLine(nextLines.head) + val currentLineYRange = new ParagraphDelimiter(currentLine.getPosition.getY, currentLine.getPosition.getHeight - 3) + val tabulation = if(nextLine.getPosition.getX - (currentLine.getPosition.getX + currentLine.getPosition.getWidth) >= 2*currentLine.getPosition.getHeight) " \t " else " " + + + // If the top of the next line is in the range, then it must be added to the current line + if(currentLineYRange === nextLine.getPosition.getY) { + // Counting of the characters having a particular font + val nextFontLength = fontLengthMap.get(nextLine.getFontID) + // If a font is always present in the map, then add to the current value the number of characters, otherwise create a new font key + val newMap = if(nextFontLength == None) fontLengthMap + ((nextLine.getFontID, (tabulation + nextLine.getText).length)) else fontLengthMap - (nextLine.getFontID) + ((nextLine.getFontID, (tabulation + nextLine.getText).length + nextFontLength.get)) + + incorporate(nextLines.tail, currentLine.addText(nextLine.setText(tabulation + nextLine.getText)), newMap) + } + else { + // Determining of the real line font + val maxFontLength = fontLengthMap.toList.map((f:(String, Int)) => f._2).max + (nextLines, currentLine.setFontID(fontLengthMap.filter((p:(String, Int)) => p._2 == maxFontLength).head._1)) + } + } + + // This is the main step of the process (see documentation). The output of the method is (newParagraph, line included?, paragraph will continue?) + def linkLine(line: XMLLine, paragraph: XMLParagraph): (XMLParagraph, Boolean, Boolean) = { + def constructEnumFormat(line: String): String = { + val enumRegex = """^([^0-9]*)?[0-9]+([^0-9]+?) """ + + if(line.length < 5) return "" + else { + val result = enumRegex.r.findFirstIn(line.take(5)) + if(result.isDefined) return """[0-9]""".r.replaceAllIn(result.get, "{d}").dropRight(1) + else return "" + } + } + + val lineCenter = ((2 * line.getPosition.getX) + line.getPosition.getWidth) / 2 + val enumFormat = constructEnumFormat(line.getText) + + // First paragraph's line (rule 1) + if(paragraph.getLines.length == 0) { + val optionContainer = if(pageCenter === lineCenter) (new XMLParagraphOptionsContainer(XMLParagraphOptions.NONE)).addOption(XMLParagraphOptions.PAGE_CENTERED) else new XMLParagraphOptionsContainer(XMLParagraphOptions.NONE) + val enumeratedOptionContainer = if(enumFormat != "") optionContainer.addOption(XMLParagraphOptions.ENUMERATION) else optionContainer + + return (new XMLParagraph(line.getFontID, line.getPosition, enumeratedOptionContainer, List(line), lineSeparator, line.getText, enumFormat), true, true) + } + + // Calculate some important parameters + val previousLineCenter = new ParagraphDelimiter(((2 * paragraph.getLines.head.getPosition.getX) + paragraph.getLines.head.getPosition.getWidth) / 2, 3) + val previousLineText = paragraph.getLines.head.getText + val previousLineTop = paragraph.getLines.head.getPosition.getY + val previousLineHeight = paragraph.getLines.head.getPosition.getHeight + val previousLineBegin = new ParagraphDelimiter(paragraph.getLines.head.getPosition.getX, 3) + val previousLineEnd = new ParagraphDelimiter(paragraph.getLines.head.getPosition.getX + paragraph.getLines.head.getPosition.getWidth, 3) + val lineBegin = line.getPosition.getX + val lineEnd = line.getPosition.getX + line.getPosition.getWidth + val lineTop = line.getPosition.getY + val lineHeight = line.getPosition.getHeight + val previousLineCapitalVersurNotDifference = """[A-Z]""".r.findAllIn(previousLineText).length - """[a-z]""".r.findAllIn(previousLineText).length + val lineCapitalVersurNotDifference = """[A-Z]""".r.findAllIn(line.getText).length - """[a-z]""".r.findAllIn(line.getText).length + + // Rule 2 + if(lineTop - previousLineTop > 2*lineHeight || !fontsContainer.getXMLFont(paragraph.getFontID).get.checkID(line.getFontID)) return (paragraph, false, false) + // Rule 3: The current line is not a title but previous is (contains only capital letters), even if the have the same font + if(lineCapitalVersurNotDifference <= 0 && previousLineCapitalVersurNotDifference > 0) return (paragraph, false, false) + // If both lines have the same enumeration format (which must be defined), then the current line is a new enumeration, hence it belongs to a new paragraph + if(paragraph.hasOption(XMLParagraphOptions.ENUMERATION) && enumFormat == paragraph.getEnumerationFormat) return (paragraph, false, false) + + + // This is the second paragraph's line + if(paragraph.getLines.length == 1) { + // Rule 4 + if(previousLineBegin === lineBegin && previousLineEnd === lineEnd && previousLineCenter === lineCenter) return (paragraph.addLine(line).addOption(XMLParagraphOptions.CENTERED).addOption(XMLParagraphOptions.JUSTIFY), true, true) + // Rule 5 + else if(previousLineBegin != lineBegin && previousLineEnd != lineEnd && previousLineCenter === lineCenter) return (paragraph.addLine(line).addOption(XMLParagraphOptions.CENTERED), true, true) + // Rule 6 + else if(previousLineEnd >= lineEnd) return (paragraph.addLine(line).addOption(XMLParagraphOptions.JUSTIFY), true, true) + } + + // Normal line (more than the second) + if(paragraph.getLines.length > 1) { + if(paragraph.hasOption(XMLParagraphOptions.CENTERED) && !paragraph.hasOption(XMLParagraphOptions.JUSTIFY)) { + // Rule 7.a + if(previousLineCenter === lineCenter) return (paragraph.addLine(line), true, true) + } + else if(!paragraph.hasOption(XMLParagraphOptions.CENTERED) && paragraph.hasOption(XMLParagraphOptions.JUSTIFY)) { + // Rule 8.a + if(previousLineBegin === lineBegin && previousLineEnd === lineEnd) return (paragraph.addLine(line), true, true) + // Rule 8.b + else if(previousLineBegin === lineBegin && previousLineEnd > lineEnd) return (paragraph.addLine(line), true, false) + } + else if(paragraph.hasOption(XMLParagraphOptions.CENTERED) && paragraph.hasOption(XMLParagraphOptions.JUSTIFY)) { + // Rule 9.a + if(previousLineBegin === lineBegin && previousLineEnd === lineEnd) return (paragraph.addLine(line), true, true) + // Rule 9.b + else if(previousLineBegin != lineBegin && previousLineEnd != lineEnd && previousLineCenter === lineCenter) return (paragraph.addLine(line).removeOption(XMLParagraphOptions.JUSTIFY), true, true) + // Rule 9.c + else if(previousLineBegin === lineBegin && previousLineEnd > lineEnd) return (paragraph.addLine(line).removeOption(XMLParagraphOptions.CENTERED), true, false) + } + } + + // For all other cases + (paragraph, false, false) + } + + + // This method applies the layout rules. + def setLayout(paragraph: XMLParagraph): XMLParagraph = { + if(pageCenter > paragraph.getPosition.getX + paragraph.getPosition.getWidth) paragraph.addOption(XMLParagraphOptions.COLUMN_LEFT) + else if(pageCenter < paragraph.getPosition.getX) paragraph.addOption(XMLParagraphOptions.COLUMN_RIGHT) + else paragraph.addOption(XMLParagraphOptions.NO_COLUMN) + } + + def constructNewParagraph(lines: NodeSeq, currentParagraph: XMLParagraph): (NodeSeq, XMLParagraph) = lines.isEmpty match{ + case true => (lines, currentParagraph) + case false => + val tempLine = createXMLLine(lines.head) + // Incorporation + val incorporation = incorporate(lines.tail, tempLine, Map((tempLine.getFontID, tempLine.getText.length))) + + val remainingLines = incorporation._1 + // Linkage + val linkage = linkLine(incorporation._2, currentParagraph) + + // If the line was added + val finalRemainingLines = if(linkage._2 == true) remainingLines else lines + + // If the paragraph continues + if(linkage._3 == true) constructNewParagraph(finalRemainingLines, linkage._1) + else (finalRemainingLines, linkage._1) + } + + // This method constructs the paragraphs. Be careful because the final list is reversed + def constructParagraphs(lines: NodeSeq, accu: List[XMLParagraph]): List[XMLParagraph] = lines.isEmpty match { + case true => accu + case false => + val construction = constructNewParagraph(lines, XMLObjectsManager.getCleanXMLParagraph(lineSeparator)) + val layoutParagraph = setLayout(construction._2.reverseLines) + constructParagraphs(construction._1, layoutParagraph :: accu) + } + + // Global page processing + def processGlobalPage(paragraphs: List[XMLParagraph]): List[XMLParagraph] = paragraphs.filterNot(p => (p.getLines.length == 1 && p.getText.length() <= 3) || p.getPosition.getX < 0 || (p.getPosition.getX + p.getPosition.getWidth) > (pagePosition.getX + pagePosition.getWidth) || p.getPosition.getY < 0 || (p.getPosition.getY + p.getPosition.getHeight) > (pagePosition.getY + pagePosition.getHeight)) + + val finalParagraphs = processGlobalPage(constructParagraphs(lines, List()).reverse) + + Some(finalParagraphs) + } +} diff --git a/src/paper/XMLParser.scala b/src/paper/XMLParser.scala new file mode 100644 index 0000000..12236b0 --- /dev/null +++ b/src/paper/XMLParser.scala @@ -0,0 +1,79 @@ +package paper +import scala.io.Source +import scala.xml.XML +import scala.xml.Elem +import scala.xml.NodeSeq +import scala.xml.TypeSymbol +import scala.util.matching.Regex.MatchIterator +import sun.nio.cs.Unicode + + +object XMLParser extends Parsers with TitleExtractor1 + with AuthorsExtractor1 + with AbstractExtractor1 + with BodyExtractor1 + with ReferencesExtractor1{ + + val extractionOrder: List[(Paper, XMLDocument, List[XMLParagraph]) => (List[XMLParagraph], Paper)] = List(extractTitle, extractAuthors, extractAbstract, extractBody, extractReferences) + + + // This method returns the xml representation of the text contained in the Source object + def getXMLObject(in: Source): Option[Elem] = { + // String generation and illegal xml characters removing + val text = in.mkString.replace("" + '\uffff', "").replace("" + "\u001f", "") + // This instruction is important, otherwise the xml file can't be deleted + in.close + + try { + // The replacement of the and tags is important because loadString sometimes generate an exception about these tags + // Of course, some information is lost, but not really an important one + Some(XML.loadString("""""".r.replaceAllIn(text, ""))) + } catch { + case _ => println("Couldn't load the XML file."); None + } + } + + + // Method for references extraction following the extraction order + def extract(extractors: List[(Paper, XMLDocument, List[XMLParagraph]) => (List[XMLParagraph], Paper)], t : (XMLDocument, Option[Paper], List[XMLParagraph])): (XMLDocument, Option[Paper], List[XMLParagraph]) = { + def extract0(extractors: List[(Paper, XMLDocument, List[XMLParagraph]) => (List[XMLParagraph], Paper)], t : (XMLDocument, Option[Paper], List[XMLParagraph])): (XMLDocument, Option[Paper], List[XMLParagraph]) = { + if(extractors.length == 0) return t + + val input = if(extractors.length == 1) t else extract0(extractors.tail, t) + val xml = input._1 + val paper = input._2 + val paragraphs = input._3 + + if(paper != None) { + // Calling the extraction method of the extractor + val extraction = extractors.head(paper.get, xml, paragraphs) + return (xml, Some(extraction._2), extraction._1) + } + + (xml, None, paragraphs) + } + + extract0(extractors.reverse, t) + } + + // The function for actually parsing a paper + def parse(in: Source) : Option[Paper] = { + val xml = getXMLObject(in) + + if(xml == None) None + else { + val cleanPaper = Paper(0, 0, Title(""), Nil, Abstract("Not saved"), Body("Not saved"), List(), Map.empty, List()) + val xmlDocument = XMLObjectsManager.constructXMLDocument(xml.get, "\n") + + // print + //xmlDocument.get.getParagraphs.foreach((p : XMLParagraph) => println(p.getText + "\n" + p.getOptionsValue + "\n" + p.getEnumerationFormat + "\n\n\n")) + + if(xmlDocument == None) return None + val paper = extract(extractionOrder, (xmlDocument.get, Some(cleanPaper), xmlDocument.get.getParagraphs)) + + if(paper._2 == None) None + else Some(paper._2.get.setMeta("parsed" -> "yes")) + } + } + +} diff --git a/src/paper/XMLScheduleParser.scala b/src/paper/XMLScheduleParser.scala index 632c03e..610187d 100644 --- a/src/paper/XMLScheduleParser.scala +++ b/src/paper/XMLScheduleParser.scala @@ -1,4 +1,5 @@ package paper +import java.io.File trait XMLScheduleParser { @@ -7,19 +8,30 @@ trait XMLScheduleParser { // Overall function that loads the xml schedule and returns the papers with the extra data def getXMLSchedule(paperPos : String, papers : List[Paper]) : List[Paper] = { - + println("BEGIN OF XML SCHEDULING") + val path = paperPos + Paths.sep + "schedule.xml" + + // Check if the schedule exists + if (!(new File(path)).exists()) throw new Exception("No file called schedule.xml exists in papers path"); + + // If no paper exists, load from cache + var loadedPapers = papers + if (papers == List()) loadedPapers = CacheLoader.load(paperPos, Cache.parsed) + // Parse schedule - val xml : Map[Int, Elem] = parse(paperPos) + val xml : Map[Int, Elem] = parse(path) - // match scedule with papers - return matchXML(xml, papers); + println("END OF XML SCHEDULING") + // match schedule with papers + return matchXML(xml, loadedPapers) } + // Function for taking care of parsing the xml def parse(paperPos : String) : Map[Int, Elem] = { // Load schedule file - val schedule : Elem = XML.loadFile(paperPos + "/schedule.xml") + val schedule : Elem = XML.loadFile(paperPos) // Initialize Map var data : Map[Int, Elem] = Map.empty @@ -46,36 +58,40 @@ trait XMLScheduleParser { // Function for putting the xml in the right paper def matchXML(xml : Map[Int, Elem], papers : List[Paper]) : List[Paper]= { + def apply(p: Paper, data: Option[Elem] ): Paper = { + // Get resulting paper + val result = setXMLData(data, p) + + // Save result + Cache.save(result, Cache.scheduled) + + // Return result + result + } + // Loop through all papers and add the xml elements the appropriate one - return for (p <- papers) yield xml.get(p.id) match { - case None => println("No schedule data for paper with id: " + p.id); p - case Some(data) => { - // Get resulting paper - val result = setXMLData(data, p) - - // Save result - Cache.save(result, Cache.scheduled) - - // Return result - result - } + return for (p <- papers) yield { + val xmlObject = xml.get(p.id) + if (xmlObject == None) { println("No schedule data for paper with id: " + p.id); p} + else apply(p, xmlObject) } } // Putting the xml in a paper - def setXMLData(xml : Elem, paper : Paper) : Paper = { - paper.setMeta("xmldate" -> getDate(xml)) - .setMeta("xmlroom" -> getRoom(xml \\ "room")) - .setMeta("xmlsession" -> (xml \\ "sess").text) - .setMeta("xmlstarttime" -> (xml \\ "starttime").text) - .setMeta("xmlendtime" -> (xml \\ "endtime").text) - .setMeta("xmlpaperid" -> (xml \\ "paperid").text) - .setMeta("xmlsessionid" -> (xml \\ "sessionid").text) - .setMeta("xmlpapertitle" -> (xml \\ "papertitle").text) - .setMeta("xmlabstract" -> (xml \\ "abstract").text) - .setMeta("xmlauthors" -> getAuthors(xml \\ "authors").mkString(", ")) - .setTitle((xml \\ "papertitle").text) - .setAuthors(getAuthors(xml \\ "authors").map(a => Author(formatAuthors(a)))) + def setXMLData(xmlObject : Option[Elem], paper : Paper) : Paper = { + val xml = xmlObject.get + paper.setMeta("xmldate" -> getDate(xml)) + .setMeta("xmlroom" -> getRoom(xml \\ "room")) + .setMeta("xmlsession" -> (xml \\ "sess").text) + .setMeta("xmlstarttime" -> (xml \\ "starttime").text) + .setMeta("xmlendtime" -> (xml \\ "endtime").text) + .setMeta("xmlpaperid" -> (xml \\ "paperid").text) + .setMeta("xmlsessionid" -> (xml \\ "sessionid").text) + .setMeta("xmlpapertitle" -> (xml \\ "papertitle").text) + //.setMeta("xmlabstract" -> (xml \\ "abstract").text) + .setMeta("xmlauthors" -> getAuthors(xml \\ "authors").mkString(", ")) + .setTitle(new Title((xml \\ "papertitle").text)) + .setAuthors(getAuthors(xml \\ "authors").map(a => Author(formatAuthors(a)))) } // Converts an authors XML note to string diff --git a/tools/linux/pdfToxmlConverter.txt b/tools/linux/pdfToxmlConverter.txt new file mode 100644 index 0000000..ab7d473 --- /dev/null +++ b/tools/linux/pdfToxmlConverter.txt @@ -0,0 +1 @@ +If this tool doesn't work, please install pdftohtml \ No newline at end of file diff --git a/tools/windows/pdfTotxtConverter.exe b/tools/windows/pdfTotxtConverter.exe new file mode 100644 index 0000000..82ffa02 Binary files /dev/null and b/tools/windows/pdfTotxtConverter.exe differ diff --git a/tools/windows/pdfToxmlConverter.exe b/tools/windows/pdfToxmlConverter.exe new file mode 100644 index 0000000..e0d2d5a Binary files /dev/null and b/tools/windows/pdfToxmlConverter.exe differ