func StartClient(c chan string, bar *pb.ProgressBar) {
	readClient, writeClient := http.Client{}, http.Client{}
	for file := range c {
		file = escapeFile(file)
		read := readUrl + file
		write := writeUrl + file
		// get
		resp, err := readClient.Get(read)
		if err != nil {
			panic(err)
		}
		getMd5 := resp.Header.Get("X-Ae-Md5")
		length := resp.ContentLength

		// put
		req, err := http.NewRequest(*method, write, resp.Body)
		if err != nil {
			panic(err)
		}
		req.ContentLength = length
		wres, err := writeClient.Do(req)
		if err != nil {
			panic(err)
		}
		if wres.StatusCode != http.StatusConflict {
			putMd5 := wres.Header.Get("X-Ae-Md5")
			if putMd5 != getMd5 {
				fmt.Printf("ERROR! MD5 not equals: %s vs %s (%s)\n", getMd5, putMd5, file)
			}
		}
		wres.Body.Close()
		resp.Body.Close()
		bar.Increment()
	}
}

func httpWorker(jobs <-chan int, hosts []host, bar *pb.ProgressBar) {
	for job := range jobs {
		h := &hosts[job]
		timeout, err := time.ParseDuration(*httpClientTimeout)
		if err != nil {
			log.Fatal(err)
		}
		client := http.Client{
			Timeout: timeout,
		}
		url := fmt.Sprintf("%s://%s:%d/xmldata?item=All", *scheme, hosts[job].ip.String(), *port)
		// Anon function for proper wg.Done() call
		func() {
			defer bar.Increment()
			defer wg.Done()
			res, err := client.Get(url)
			if err != nil {
				h.Err += err.Error()
				return
			}
			xmldata, err := ioutil.ReadAll(res.Body)
			defer res.Body.Close()
			if err != nil {
				h.Err += err.Error()
				return
			}
			h.XMLErr = xml.Unmarshal(xmldata, h)
		}()
	}
}

func (g *Game) getInitTweets(aliveNum int) {
	var progress *pb.ProgressBar
	if !g.debug {
		progress = pb.StartNew(aliveNum)
	}
	for y := 0; y < g.field.SizeY; y++ {
		for x := 0; x < g.field.SizeX; x++ {
			if g.field.Points[y][x].IsAlive {
				tweet := <-g.tweetCh
				if g.debug {
					emoji.Printf(":bird:Points[%v][%v]: %v\n", x, y, tweet)
				}
				g.field.Points[y][x].Str = tweet
				if !g.debug {
					progress.Increment()
				}
			}
		}
	}
	if g.debug {
		emoji.Println(":smile::smile::smile:Collected initial tweets:smile::smile::smile:")
	} else {
		e := emoji.Sprint(":smile::smile::smile:")
		progress.FinishPrint(e + "Collected initial tweets" + e)
	}
}

func work(wg *sync.WaitGroup, bar *pb.ProgressBar, queue chan *Track) {
	defer wg.Done()

	for track := range queue {
		track.Download()
		bar.Increment()
	}
}

func main() {

	var progress *pb.ProgressBar

	page := 0
	category := "bugs"

	for {

		printf("Get page: %d", page)
		tickets, _, err := sfClient.Tracker.Info(category)

		if err != nil {
			log.Fatal(err)
		}

		if ghMilestones == nil {
			createMilestones(tickets)
			getMilestones()
		}

		if progress == nil {
			log.Println("Creating tickets")
			progress = pb.StartNew(tickets.Count)
		}

		if len(tickets.Tickets) == 0 {
			break
		}

		for _, ticket := range tickets.Tickets {
			ticket, _, err := sfClient.Tracker.Get(category, ticket.TicketNum)

			if err != nil {
				log.Fatal(err)
			}

			sfTicketToGhIssue(ticket, category)

			progress.Increment()
			time.Sleep(time.Millisecond * sleepTime)
		}

		page += 1
	}

	progress.FinishPrint("All tickets imported")
}

func HostMerge(List []Host, ShowBar bool) []string {
	count := 0
	filterList := []string{""}
	length := len(List)
	var bar *pb.ProgressBar
	if ShowBar == true {
		bar = pb.StartNew(length)
		bar.SetMaxWidth(80)
	}
	for _, Host := range List {
		length = len(filterList[count])
		if length == 0 {
			filterList[count] = Host.Hostname
		} else if length+Host.length() <= 255 && length != 0 {
			filterList[count] += "|"
			filterList[count] += Host.Hostname
		} else {
			count++
			filterList = append(filterList, Host.Hostname)
			// filterList[count] = Ref.Referrer
		}
		if ShowBar == true {
			bar.Increment()
			time.Sleep(time.Millisecond * 50)
		}
	}
	if ShowBar == true {
		bar.Finish()
	}
	return filterList
}

func customizeBar(bar *pb.ProgressBar) {
	bar.ShowCounters = true
	bar.ShowTimeLeft = false
	bar.ShowSpeed = true
	bar.SetMaxWidth(80)
	bar.SetUnits(pb.U_BYTES)
}

func main() {
	opts, args := parseFlags()
	conv := cbconvert.NewConvertor(opts)

	var bar *pb.ProgressBar

	c := make(chan os.Signal, 3)
	signal.Notify(c, os.Interrupt, syscall.SIGHUP, syscall.SIGTERM)
	go func() {
		for _ = range c {
			fmt.Fprintf(os.Stderr, "Aborting\n")
			os.RemoveAll(conv.Workdir)
			os.Exit(1)
		}
	}()

	if _, err := os.Stat(opts.Outdir); err != nil {
		os.MkdirAll(opts.Outdir, 0777)
	}

	files := conv.GetFiles(args)

	if opts.Cover || opts.Thumbnail {
		if !opts.Quiet {
			bar = pb.New(conv.Nfiles)
			bar.ShowTimeLeft = false
			bar.Start()
		}
	}

	for _, file := range files {
		stat, err := os.Stat(file)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error Stat: %v\n", err.Error())
			continue
		}

		if opts.Cover {
			conv.ExtractCover(file, stat)
			if !opts.Quiet {
				bar.Increment()
			}
			continue
		} else if opts.Thumbnail {
			conv.ExtractThumbnail(file, stat)
			if !opts.Quiet {
				bar.Increment()
			}
			continue
		}

		conv.ConvertComic(file, stat)
	}
}

// download a file with the HTTP/HTTPS protocol showing a progress bar. The destination file is
// always overwritten.
func download(rawurl string, destinationPath string) {
	tempDestinationPath := destinationPath + ".tmp"

	destination, err := os.Create(tempDestinationPath)
	if err != nil {
		log.Fatalf("Unable to open the destination file: %s", tempDestinationPath)
	}
	defer destination.Close()

	response, err := customGet(rawurl)
	if err != nil {
		log.Fatalf("Unable to open a connection to %s", rawurl)
	}
	defer response.Body.Close()

	if response.StatusCode != http.StatusOK {
		log.Fatalf("Unexpected HTTP response code. Wanted 200 but got %d", response.StatusCode)
	}

	var progressBar *pb.ProgressBar

	contentLength, err := strconv.Atoi(response.Header.Get("Content-Length"))
	if err == nil {
		progressBar = pb.New(int(contentLength))
	} else {
		progressBar = pb.New(0)
	}
	defer progressBar.Finish()

	progressBar.ShowSpeed = true
	progressBar.SetRefreshRate(time.Millisecond * 1000)
	progressBar.SetUnits(pb.U_BYTES)
	progressBar.Start()

	writer := io.MultiWriter(destination, progressBar)

	io.Copy(writer, response.Body)
	destination.Close()
	os.Rename(tempDestinationPath, destinationPath)
}

func runClusterBackup(args *docopt.Args) error {
	client, err := getClusterClient()
	if err != nil {
		return err
	}

	var bar *pb.ProgressBar
	var progress backup.ProgressBar
	if term.IsTerminal(os.Stderr.Fd()) {
		bar = pb.New(0)
		bar.SetUnits(pb.U_BYTES)
		bar.ShowBar = false
		bar.ShowSpeed = true
		bar.Output = os.Stderr
		bar.Start()
		progress = bar
	}

	var dest io.Writer = os.Stdout
	if filename := args.String["--file"]; filename != "" {
		f, err := os.Create(filename)
		if err != nil {
			return err
		}
		defer f.Close()
		dest = f
	}

	fmt.Fprintln(os.Stderr, "Creating cluster backup...")

	if err := backup.Run(client, dest, progress); err != nil {
		return err
	}
	if bar != nil {
		bar.Finish()
	}
	fmt.Fprintln(os.Stderr, "Backup complete.")

	return nil
}

func scan(ips []string, out chan ILOInfo, bar *pb.ProgressBar, wg *sync.WaitGroup) {
	for _, host := range ips {
		if IsOpen(host, iloPort) {
			srvName := ""
			iloName := ""
			info, err := requestInfo(host)
			if err != nil {
				fmt.Println(err)
			}
			if match, _ := regexp.MatchString("iLO (3|4|5)", info.HW); match {
				srvName, iloName, _ = requestServerName(host)
			} else {
				srvName, iloName, _ = requestServerNameV2(host)
			}
			info.ServerName = srvName
			info.IloName = iloName
			out <- *info
		}
		bar.Increment()
	}
	wg.Done()
}

// processSystemArchives processes archives for given system
func (h *Harvester) processSystemArchives(s *system.System, archives []string) error {
	var bar *pb.ProgressBar

	nb := len(archives)

	// extract archives
	if !s.Options.Quiet {
		fmt.Printf("[%s] Extracting %v archive(s)\n", s.Infos.Name, nb)

		if !s.Options.Debug {
			bar = pb.StartNew(nb)
			bar.ShowCounters = true
			bar.ShowPercent = false
			bar.ShowTimeLeft = true
			bar.SetMaxWidth(80)
		}
	}

	for _, archive := range archives {
		if !s.Options.Quiet && !s.Options.Debug {
			bar.Increment()
		}

		if err := s.ProcessArchive(archive, h.Options.Output); err != nil {
			return err
		}
	}

	if !s.Options.Quiet && !s.Options.Debug {
		bar.Finish()
		fmt.Printf("[%s] Processed %v files (skipped: %v)\n", s.Infos.Name, s.Processed, s.Skipped)
	}

	fmt.Printf("[%s] Selected %v games\n", s.Infos.Name, len(s.Games))

	return nil
}

func uploadItem(c *cli.Context, hash string, item client.IndexItem, bar *pb.ProgressBar, ch chan error) {
	data, err := ioutil.ReadFile(item.Name)

	if err != nil {
		ch <- err
		return
	}

	for i := 0; i < 3; i++ {
		err = rackClient(c).IndexUpload(hash, data)

		if err != nil {
			continue
		}

		bar.Add(item.Size)

		ch <- nil
		return
	}

	ch <- fmt.Errorf("max 3 retries on upload")
	return
}

func indexAndSaveHits(ts *index.TokenSetSearcher, hits []HitInfo, idxs []int, saveFullHit func(*HitInfo) error) error {
	rank := 0
	var bar *pb.ProgressBar
	if terminal.IsTerminal(int(os.Stdout.Fd())) {
		bar = pb.New(len(idxs))
		bar.Start()
	}
	for i := range idxs {
		hit := &hits[idxs[i]]
		if i > 0 && hit.StaticScore < hits[idxs[i-1]].StaticScore {
			rank = i
		}
		hit.StaticRank = rank

		if err := saveFullHit(hit); err != nil {
			return err
		}

		var desc, readme string
		desc, hit.Description = hit.Description, ""
		readme, hit.ReadmeData = hit.ReadmeData, ""
		hit.Imported = nil
		hit.TestImported = nil

		var nameTokens stringsp.Set
		nameTokens = AppendTokens(nameTokens, []byte(hit.Name))

		var tokens stringsp.Set
		tokens.Add(nameTokens.Elements()...)
		tokens = AppendTokens(tokens, []byte(hit.Package))
		tokens = AppendTokens(tokens, []byte(desc))
		tokens = AppendTokens(tokens, []byte(readme))
		tokens = AppendTokens(tokens, []byte(hit.Author))
		for _, word := range hit.Exported {
			AppendTokens(tokens, []byte(word))
		}
		ts.AddDoc(map[string]stringsp.Set{
			IndexTextField: tokens,
			IndexNameField: nameTokens,
			IndexPkgField:  stringsp.NewSet(hit.Package),
		}, *hit)
		if bar != nil {
			bar.Increment()
		}
	}
	if bar != nil {
		bar.FinishPrint("Indexing finished!")
	}
	DumpMemStats()
	return nil
}

func StoreRedirects(db *sql.DB, redirs []wikidump.Redirect,
	bar *pb.ProgressBar) error {

	counts := make([]linkCount, 0)

	var titleId, old, del, delTitle, insTitle, ins, update *sql.Stmt
	tx, err := db.Begin()
	if err == nil {
		titleId, err = tx.Prepare(`select id from titles where title = ?`)
	}
	if err == nil {
		old, err = tx.Prepare(
			`select ngramhash, count from linkstats where targetid = ?`)
	}
	if err == nil {
		del, err = tx.Prepare(`delete from linkstats where targetid = ?`)
	}
	if err == nil {
		delTitle, err = tx.Prepare(`delete from titles where id = ?`)
	}
	if err == nil {
		insTitle, err = tx.Prepare(
			`insert or ignore into titles values (NULL, ?)`)
	}
	if err == nil {
		ins, err = tx.Prepare(
			`insert or ignore into linkstats values
			 (?, (select id from titles where title = ?), 0)`)
	}
	if err == nil {
		update, err = tx.Prepare(
			`update linkstats set count = count + ?
			 where targetid = (select id from titles where title = ?)
			       and ngramhash = ?`)
	}
	if err != nil {
		return err
	}

	for _, r := range redirs {
		if bar != nil {
			bar.Increment()
		}

		var fromId int
		err := titleId.QueryRow(r.Title).Scan(&fromId)
		if err == sql.ErrNoRows { // No links to this redirect.
			continue
		} else if err != nil {
			return err
		}

		rows, err := old.Query(fromId)
		if err != nil {
			return err
		}

		// SQLite won't let us INSERT or UPDATE while doing a SELECT.
		for counts = counts[:0]; rows.Next(); {
			var count float64
			var hash int64
			rows.Scan(&hash, &count)
			counts = append(counts, linkCount{hash, count})
		}
		rows.Close()
		err = rows.Err()

		if err == nil {
			_, err = del.Exec(fromId)
		}
		if err == nil {
			_, err = delTitle.Exec(fromId)
		}

		if err != nil {
			return err
		}

		for _, c := range counts {
			if err == nil {
				_, err = insTitle.Exec(r.Target)
			}
			if err == nil {
				_, err = ins.Exec(c.hash, r.Target)
			}
			if err == nil {
				_, err = update.Exec(c.count, r.Target, c.hash)
			}
		}
		if err != nil {
			return err
		}
	}
	err = tx.Commit()
	return err
}

// PostMultipartP posts a multipart message in the MIME internet format with a callback function with a string stating the upload Progress.
func (c *Client) PostMultipartP(path string, files map[string][]byte, params Params, out interface{}, callback func(s string)) error {
	body := &bytes.Buffer{}

	writer := multipart.NewWriter(body)

	for name, source := range files {
		part, err := writer.CreateFormFile(name, "source.tgz")
		if err != nil {
			return err
		}

		_, err = io.Copy(part, bytes.NewReader(source))
		if err != nil {
			return err
		}
	}

	for name, value := range params {
		writer.WriteField(name, value)
	}

	err := writer.Close()
	if err != nil {
		return err
	}

	var bodyReader io.Reader
	bodyReader = body

	var bar *pb.ProgressBar

	if callback != nil {
		bar = pb.New(body.Len()).SetUnits(pb.U_BYTES)
		bar.NotPrint = true
		bar.ShowBar = false
		bar.Callback = callback

		bar.Start()
		bodyReader = bar.NewProxyReader(body)
	}

	req, err := c.request("POST", path, bodyReader)

	if err != nil {
		return err
	}

	req.SetBasicAuth("convox", string(c.Password))

	req.Header.Set("Content-Type", writer.FormDataContentType())

	res, err := c.client().Do(req)

	if err != nil {
		return err
	}

	defer res.Body.Close()

	if err := responseError(res); err != nil {
		return err
	}

	data, err := ioutil.ReadAll(res.Body)

	if err != nil {
		return err
	}

	if out != nil {
		err = json.Unmarshal(data, out)

		if err != nil {
			return err
		}
	}

	if callback != nil {
		bar.Finish()
	}

	return nil
}

func (ctx *Context) Load() error {
	var bar *pb.ProgressBar
	if Verbose {
		log.Println("loading database")
	}
	db, err := LoadDB(ctx.DatabaseName, *decompress)
	if os.IsNotExist(err) {
		log.Printf("database not found")
		return nil
	} else if err != nil {
		log.Printf("error loading database: %s", err)
		return err
	}

	if Verbose {
		log.Println("loading teams")
		bar = pb.StartNew(len(db.Teams))
	}
	for _, team := range db.Teams {
		ctx.AddTeam(team)
		if Verbose {
			bar.Increment()
		}
	}
	if Verbose {
		bar.Finish()
	}

	if Verbose {
		log.Println("loading match history")
		bar = pb.StartNew(len(db.Outcomes))
	}
	loadedOutcomes := map[string]*Outcome{}
	for _, outcome := range db.Outcomes {
		loadedOutcomes[outcome.ID] = outcome
		if Verbose {
			bar.Increment()
		}
	}
	if Verbose {
		bar.Finish()
	}

	if Verbose {
		bar = pb.StartNew(len(db.Matches))
	}
	for _, match := range db.Matches {
		outcome, ok := loadedOutcomes[match.OutcomeID]
		if !ok {
			log.Panicf("corrupted history %q", match.ID)
		}
		ctx.AddMatch(match, outcome)
		if Verbose {
			bar.Increment()
		}
	}
	if Verbose {
		bar.Finish()
	}
	return nil
}

func install(l, version string) error {
	var currentStep lang.Step
	var bar *pb.ProgressBar
	var process *Process
	fmt.Printf("Installing %[email protected]%s\n", l, version)
	err := service.Install(l, version, binaryFlag, func(step lang.Step, progress, total int64) {

		if currentStep != step {
			if bar != nil {
				bar.NotPrint = true
				bar.Finish()
				fmt.Printf(ascii2.EraseLine)
				bar = nil
			}

			if process != nil {
				process.Done("")
				process = nil
			}

			if total > 0 {
				bar = pb.New64(total).Prefix("  " + stepToMsg(step) + "\t\t")
				bar.SetWidth(40)
				bar.ShowCounters = false
				//fmt.Printf("%s\n", step)
				//bar.NotPrint = true
				bar.Start()
				currentStep = step

			} else {
				process := &Process{Msg: stepToMsg(step) + "\t\t"}
				process.Start()
			}

		}
		if bar != nil {
			bar.Set64(progress)
		}

	})

	if bar != nil {
		bar.NotPrint = true
		bar.Finish()

		fmt.Printf(ascii2.EraseLines(2) + ascii2.EraseLine + fmt.Sprintf("  %s installed", l))
	}

	if process != nil {
		process.Done("\n")
	}
	//fmt.Printf(ascii2.EraseLine + ascii2.CursorUp(1) + ascii2.EraseLine)

	if err != nil {
		fmt.Printf("Could not install %[email protected]%s: \n  %s\n", l, version, err.Error())
	} else {
		fmt.Printf("  %[email protected]%s installed!\n\n", l, version)
	}

	return err
}

func scanWords(wordpath string, tupleLength int, makeLookup bool) (words map[string]int, tuples map[string]string, wordsLookup map[int]string, tuplesLookup map[string][]int) {
	totalLines := lineCount(wordpath)

	inFile, _ := os.Open(wordpath)
	defer inFile.Close()
	scanner := bufio.NewScanner(inFile)
	scanner.Split(bufio.ScanLines)

	// initialize
	words = make(map[string]int)
	tuples = make(map[string]string)
	wordsLookup = make(map[int]string)
	tuplesLookup = make(map[string][]int)

	numTuples := 0
	numWords := 0
	lineNum := 0
	var bar *pb.ProgressBar
	if VERBOSE {
		fmt.Println("Parsing subsets...")
		bar = pb.StartNew(totalLines)
	}
	for scanner.Scan() {
		if VERBOSE {
			bar.Increment()
		}
		lineNum++
		s := strings.TrimSpace(scanner.Text())

		_, ok := words[s]
		if ok == false {
			if makeLookup {
				wordsLookup[numWords] = s
			} else {
				words[s] = numWords
			}

			partials := getPartials(s, tupleLength)
			for i := 0; i < len(partials); i++ {
				_, ok := tuples[partials[i]]
				if makeLookup {
					_, ok = tuplesLookup[partials[i]]
				}
				if ok == false {
					if makeLookup {
						tuplesLookup[partials[i]] = append([]int{}, numWords)
					} else {
						tuples[partials[i]] = strconv.Itoa(numWords)
					}
					numTuples++
				} else {
					if makeLookup {
						tuplesLookup[partials[i]] = append(tuplesLookup[partials[i]], numWords)
					} else {
						tuples[partials[i]] += " " + strconv.Itoa(numWords)
					}
				}
			}

			numWords++
		}

	}
	if VERBOSE {
		bar.FinishPrint("Finished parsing subsets")
	}
	return
}

func importCSV(filename string, connStr string, schema string, tableName string, ignoreErrors bool, skipHeader bool, fields string, delimiter string) error {

	db, err := connect(connStr, schema)
	if err != nil {
		return err
	}
	defer db.Close()

	var reader *csv.Reader
	var bar *pb.ProgressBar
	if filename != "" {
		file, err := os.Open(filename)
		if err != nil {
			return err
		}
		defer file.Close()

		bar = NewProgressBar(file)
		reader = csv.NewReader(io.TeeReader(file, bar))
	} else {
		reader = csv.NewReader(os.Stdin)
	}

	reader.Comma, _ = utf8.DecodeRuneInString(delimiter)
	reader.LazyQuotes = true

	columns, err := parseColumns(reader, skipHeader, fields)
	if err != nil {
		return err
	}

	reader.FieldsPerRecord = len(columns)

	i, err := NewCSVImport(db, schema, tableName, columns)
	if err != nil {
		return err
	}

	var success, failed int
	if filename != "" {
		bar.Start()
		err, success, failed = copyCSVRows(i, reader, ignoreErrors, delimiter, columns)
		bar.Finish()
	} else {
		err, success, failed = copyCSVRows(i, reader, ignoreErrors, delimiter, columns)
	}

	if err != nil {
		lineNumber := success + failed
		if !skipHeader {
			lineNumber++
		}
		return errors.New(fmt.Sprintf("line %d: %s", lineNumber, err))
	} else {
		fmt.Println(fmt.Sprintf("%d rows imported into %s.%s", success, schema, tableName))

		if ignoreErrors && failed > 0 {
			fmt.Println(fmt.Sprintf("%d rows could not be imported into %s.%s and have been written to stderr.", failed, schema, tableName))
		}

		return i.Commit()
	}
}