//设置package包名称以及导入依赖的类
package uk.pkerrigan.dmarcparser

import java.io.ByteArrayInputStream
import java.nio.charset.CodingErrorAction
import java.util.Properties
import java.util.zip.{GZIPInputStream, ZipInputStream}
import javax.activation.DataSource
import javax.mail.Session
import javax.mail.internet.MimeMessage
import scala.collection.JavaConverters._

import org.apache.commons.mail.util.MimeMessageParser
import uk.pkerrigan.dmarcparser.report.Feedback

import scala.io._

class EmailParser(parser: ParserTrait = new Parser()) extends EmailParserTrait{

  implicit val codec = Codec("UTF-8")
  codec.onMalformedInput(CodingErrorAction.REPLACE)
  codec.onUnmappableCharacter(CodingErrorAction.REPLACE)

  def parseEmail(email: String): Option[Feedback] = {
    val s = Session.getDefaultInstance(new Properties())
    val is = new ByteArrayInputStream(email.getBytes)
    val message = new MimeMessage(s, is)
    val messageParser = new MimeMessageParser(message).parse()

    messageParser.getAttachmentList.asScala.headOption.flatMap(extract)
  }

  private def extract(a: DataSource): Option[Feedback] = a match {
    case `a` if a.getContentType.equals("application/gzip") => extractGzip(a)
    case `a` if a.getContentType.equals("application/x-gzip") => extractGzip(a)
    case `a` if a.getContentType.equals("application/zip") => extractZip(a)
    case `a` if a.getContentType.equals("application/x-zip-compressed") => extractZip(a)
    case _ => None
  }

  private def extractZip(a: DataSource): Option[Feedback] = {
    val zip = new ZipInputStream(a.getInputStream)
    zip.getNextEntry
    val rawXml = Source.fromInputStream(zip).mkString
    if (rawXml == "") None else Some(parser.parse(rawXml))
  }
  private def extractGzip(a: DataSource): Option[Feedback] = {
    val zip = new GZIPInputStream(a.getInputStream)
    val rawXml = Source.fromInputStream(zip).mkString
    if (rawXml == "") None else Some(parser.parse(rawXml))
  }
} 

//设置package包名称以及导入依赖的类
package util

import java.io.ByteArrayInputStream
import java.nio.charset.{Charset, CodingErrorAction}

import text.StringOption

import scala.collection.mutable.ListBuffer
import scala.io.{Codec, Source}
import scala.sys.process.ProcessBuilder


object ProcessBuilderUtils {
  implicit def processToProcessUtils(repr: ProcessBuilder): ProcessBuilderUtils = {
    new ProcessBuilderUtils(repr)
  }
}

class ProcessBuilderUtils(repr: ProcessBuilder) {
  def lineStream(encoding: Charset,
                 onMalformedInput: CodingErrorAction,
                 onUnmappableCharacter: CodingErrorAction,
                 replacementOpt: StringOption): Iterator[String] = {
    val lines: Iterator[String] = repr.lineStream_!.iterator
    val byteBuffer = ListBuffer.empty[Byte]
    while (lines.hasNext) {
      val line: String = lines.next.trim concat "\n"
      byteBuffer ++= line.getBytes
    }
    implicit val codec = Codec(encoding).
      onMalformedInput(onMalformedInput).
      onUnmappableCharacter(onUnmappableCharacter)
    if (replacementOpt.nonEmpty) {
      codec.decodingReplaceWith(replacementOpt.get)
    }
    Source.fromInputStream(new ByteArrayInputStream(byteBuffer.toArray)).getLines
  }
} 

//设置package包名称以及导入依赖的类
package swave.core

import java.nio.charset.{Charset, CodingErrorAction}
import swave.core.io.Bytes

package object text {

  implicit class RichBytesStreamOpsText[T, S[X] <: StreamOps[X]](val underlying: S[T]) extends AnyVal {

    def decode(charset: Charset,
               onMalformedInput: CodingErrorAction = CodingErrorAction.REPORT,
               onUnmappableCharacter: CodingErrorAction = CodingErrorAction.REPLACE)(
        implicit ev: Bytes[T]): S[T]#Repr[String] =
      underlying.via(Text.decode[T](charset, onMalformedInput, onUnmappableCharacter))

    def utf8Decode(implicit ev: Bytes[T]): S[T]#Repr[String] =
      underlying.via(Text.utf8Decode)
  }

  implicit class RichStringStreamOpsText[S <: StreamOps[String]](val underlying: S) extends AnyVal {

    def encode[T: Bytes](charset: Charset): S#Repr[T] =
      underlying.via(Text.encode(charset))

    def utf8Encode[T: Bytes]: S#Repr[T] =
      underlying.via(Text.utf8Encode)

    def lines: S#Repr[String] =
      underlying.via(Text.lines)
  }
} 

//设置package包名称以及导入依赖的类
package io.rosenberg.elff

import scala.io.Codec
import java.nio.charset.CodingErrorAction


	def getLogItems(lines: Iterator[String]): Iterator[collection.mutable.Map[String, String]] = {
		// this is to avoid encoding errors that frequently happen in large log files
		if (_fixEncodingError) {
			implicit val codec = Codec("UTF-8")
			codec.onMalformedInput(CodingErrorAction.REPLACE)
			codec.onUnmappableCharacter(CodingErrorAction.REPLACE)
		}
		return new LogItemsIterator(lines, fields)
	}
	
} 

//设置package包名称以及导入依赖的类
package com.codacy.dotnet.helpers

import java.io.InputStream
import java.nio.charset.{CodingErrorAction, StandardCharsets}
import java.nio.file.{Files, Path, StandardOpenOption}

import scala.io.Codec
import scala.util.{Failure, Try}

object ResourceHelper {

  implicit val codec = Codec("UTF-8")
  codec.onMalformedInput(CodingErrorAction.REPLACE)
  codec.onUnmappableCharacter(CodingErrorAction.REPLACE)

  def getResourceStream(path: String): Try[InputStream] = {
    Option(getClass.getClassLoader.getResource(path)).map { url =>
      Try(url.openStream())
    }.getOrElse {
      Failure(new Exception("The path provided was not found"))
    }
  }

  def writeFile(path: Path, content: String): Try[Path] = {
    Try(Files.write(
      path,
      content.getBytes(StandardCharsets.UTF_8),
      StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE
    ))
  }

} 

//设置package包名称以及导入依赖的类
package com.twitter.io

import java.nio.charset.{Charset, CharsetDecoder, CharsetEncoder, CodingErrorAction}
import java.util


  def decoder(charset: Charset): CharsetDecoder = {
    var d = decoders.get.get(charset)
    if (d == null) {
      d = charset.newDecoder()
      decoders.get.put(charset, d)
    }
    d.reset()
    d.onMalformedInput(CodingErrorAction.REPLACE)
    d.onUnmappableCharacter(CodingErrorAction.REPLACE)
  }
} 

//设置package包名称以及导入依赖的类
import scala.io.Source
import java.nio.charset.CodingErrorAction
import scala.io.Codec

object Scraper1 {

  def main(args: Array[String]): Unit = {
    implicit val codec = Codec("UTF-8")
    codec.onMalformedInput(CodingErrorAction.IGNORE)
    codec.onUnmappableCharacter(CodingErrorAction.REPLACE)
    val url = "http://blog.jungbin.kim"
    val html = Source.fromURL(url)
    val s = html.getLines.mkString("\n")
    println(s)
  }

} 

//设置package包名称以及导入依赖的类
package indexer

import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.scala.experimental.ScalaObjectMapper
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import java.nio.charset.CodingErrorAction
import scala.io.Codec
import scala.io.Source

object Utilities {
  implicit val codec = Codec("UTF-8")
  codec.onMalformedInput(CodingErrorAction.REPLACE)
  codec.onUnmappableCharacter(CodingErrorAction.REPLACE)

  val mapper = new ObjectMapper() with ScalaObjectMapper
  mapper.registerModule(DefaultScalaModule)

  val stopWordsFile = "src/main/resources/stopwords"
  lazy val stopWords = Source.fromFile(stopWordsFile).getLines.map(_.replaceAll("'", "")).toSet

  def tokenize(source: Source): Iterator[(String, Int)] = source.getLines
    .map(_.replaceAll("""(?<=\w)[',](?=\w)""", ""))
    .map(_.replaceAll("""(?<=\d)[.](?=\d)""", "__TOKENZ__DECML__"))
    .flatMap(_.split("""\W+"""))
    .filterNot(_ == "")
    .map(_.replaceAll("__TOKENZ__DECML__", "."))
    .map(_.toLowerCase)
    .zipWithIndex
    .filterNot { case (token, index) => !token.matches(""".*[a-z].*""") && token.size > 4 }
    .filterNot { case (token, index) => token.size < 3 || token.size > 20 }
    .filterNot { case (token, index) => stopWords(token) }

  def nGrams(n: Integer, source: Source): Iterator[(String, Int)] = tokenize(source)
    .sliding(n)
    .filter { case item => item.last._2 - item.head._2 == n - 1 }
    .filter(_.length == n)
    .map(_.unzip)
    .map { case (tokens, pos :: _) => (tokens.mkString(" "), pos)  }

  def frequencies(tokens: Iterator[(String, Int)]): Iterator[(String, Int)] = tokens
    .foldLeft(Map[String, Int]()) { case (acc, (token, _)) =>
      acc + (token -> (acc.getOrElse(token, 0) + 1))
    }
    .toIterator

  def time[R](block: => R): R = {
    val t0 = System.nanoTime()
    val result = block    // call-by-name
    val t1 = System.nanoTime()
    println("Elapsed time: " + (t1 - t0) + "ns")
    result
  }
} 

//设置package包名称以及导入依赖的类
package util.process

import java.io.ByteArrayInputStream
import java.nio.charset.{Charset, CodingErrorAction}

import text.StringOption

import scala.io.{Codec, Source}
import scala.sys.process.ProcessBuilder


class ProcessBuilderUtils(repr: ProcessBuilder) {
  def lineStream(encoding: Charset,
                 onMalformedInput: CodingErrorAction,
                 onUnmappableCharacter: CodingErrorAction,
                 replacementOpt: StringOption): Iterator[String] = {
    implicit val codec = Codec(encoding).
      onMalformedInput(onMalformedInput).
      onUnmappableCharacter(onUnmappableCharacter)
    if (replacementOpt.nonEmpty) {
      codec.decodingReplaceWith(replacementOpt.get)
    }
    Source.fromInputStream(
      new ByteArrayInputStream(
        repr.
          lineStream_!.
          iterator.
          mkString("\n").
          getBytes
      )
    ).getLines
  }
} 

//设置package包名称以及导入依赖的类
package text.analyzer.dep.chapas

import java.nio.charset.{CodingErrorAction, StandardCharsets}

import text.StringNone
import text.analyzer.dep.cabocha.CaboCha
import text.normalizer.NormalizedString
import util.{SentenceUnitReconstructorConfig, ShowcaseCache}
import util.process.EchoProcess
import util.process.ProcessBuilderUtils._


import scala.sys.process.Process


object Showcase extends PasAnalyzer with EchoProcess with ShowcaseCache {
  private val showcaseCmd: String = {
    SentenceUnitReconstructorConfig.showcase match {
      case Some(c) => c
      case None => throw new RuntimeException("Showcase command was not found.")
    }
  }

  override def analyze(sentence: NormalizedString): Iterator[String] = {
    CaboCha.cabochaLatticeJumanCmd(sentence).#|(Process(showcaseCmd)).lineStream(
      StandardCharsets.UTF_8,
      CodingErrorAction.IGNORE,
      CodingErrorAction.IGNORE,
      StringNone
    )
  }

  override def cabochaLattice(sentence: NormalizedString): Iterator[String] = {
    CaboCha.cabochaLatticeJumanCmd(sentence).lineStream(
      StandardCharsets.UTF_8,
      CodingErrorAction.IGNORE,
      CodingErrorAction.IGNORE,
      StringNone
    )
  }

  override def cabochaTree(sentence: NormalizedString): Iterator[String] = {
    CaboCha.cabochaTreeJumanCmd(sentence).lineStream(
      StandardCharsets.UTF_8,
      CodingErrorAction.IGNORE,
      CodingErrorAction.IGNORE,
      StringNone
    )
  }

  override final protected[this] def textToDoc(normalizedSentencesOption: NormalizedString): Doc =
    (if (cacheEnable) getDocCache(normalizedSentencesOption) else None) getOrElse
      this.parser.parse(analyze(normalizedSentencesOption))

  override val parser: ShowcaseNtcParser.type = ShowcaseNtcParser
} 

//设置package包名称以及导入依赖的类
package ir.fulltext.indri

import java.nio.charset.{CodingErrorAction, StandardCharsets}
import java.nio.file.Path

import text.StringNone
import util.Config

import scala.collection.mutable.ListBuffer
import scala.sys.process.Process


class IndriIndex(inputPath: Path, indexPath: Path) {
  private def command: Seq[String] = {
    Seq[String](
      "IndriBuildIndex",
      "-field.name=TITLE",
      "-memory=".concat(Config.indriMemory),
      "-corpus.path=".concat(inputPath.toAbsolutePath.toString),
      "-corpus.class=trectext",
      "-index=".concat(indexPath.toAbsolutePath.toString)
    )
  }

  def index(): Unit = {
    val buffer: ListBuffer[String] = ListBuffer.empty[String]
    command.foreach(buffer.+=)
    import util.process.ProcessBuilderUtils._
    Process(buffer.result).lineStream(
      StandardCharsets.UTF_8,
      CodingErrorAction.IGNORE,
      CodingErrorAction.IGNORE,
      StringNone
    ).foreach(println)
  }
} 

//设置package包名称以及导入依赖的类
package us.feliscat.ir.fulltext.indri

import java.nio.charset.{CodingErrorAction, StandardCharsets}
import java.nio.file.Path

import us.feliscat.text.StringNone
import us.feliscat.util.LibrariesConfig
import us.feliscat.util.process._

import scala.collection.mutable.ListBuffer
import scala.concurrent.duration._
import scala.sys.process.Process


class IndriIndexer(inputPath: Path, indexPath: Path) {
  private def command: Seq[String] = {
    "IndriBuildIndex" ::
    "-field.name=TITLE" ::
    "-memory=".concat(LibrariesConfig.indriMemory) ::
    "-corpus.path=".concat(inputPath.toAbsolutePath.toString) ::
    "-corpus.class=trectext" ::
    "-index=".concat(indexPath.toAbsolutePath.toString) :: Nil
  }

  def index(): Unit = {
    val buffer = ListBuffer.empty[String]
    command.foreach(buffer.+=)
    Process(buffer.result).lineStream(
        StandardCharsets.UTF_8,
        CodingErrorAction.IGNORE,
        CodingErrorAction.IGNORE,
        StringNone,
        LibrariesConfig.indriBuildIndexTimeout.minute
      ).foreach(println)
  }
}