#!/usr/bin/env bash

set -o errexit
set -o errtrace

SOURCEURL='http://s3.amazonaws.com/alexa-static/top-1m.csv.zip'

cargo build --release

tmp_directory="$(mktemp -d)"

exit_handler() {
	rm -r "$tmp_directory"
}

trap exit_handler EXIT

curl \
	--disable \
	--location \
	--max-time 60 \
	--output "$tmp_directory/top.csv.zip" \
	"$SOURCEURL"

unzip "$tmp_directory/top.csv.zip" -d "$tmp_directory"

xsv select 2 "$tmp_directory/top-1m.csv" | sort -u > "$tmp_directory/top.txt"

./target/release/sectxt --threads 50 --timeout 5 --quiet < "$tmp_directory/top.txt"
