phishtank_api/stats.sh

#!/bin/bash - 
#===============================================================================
#
#          FILE: stats.sh
#
#         USAGE: ./stats.sh
#
#   DESCRIPTION: Prints all needed statistics about the collected *.csv files
#
#       OPTIONS: ---
#  REQUIREMENTS: wc, cat, uniq, sort
#          BUGS: ---
#         NOTES: ---
#        AUTHOR: Francesco Emanuel Bennici (l0nax), benniciemanuel78@gmail.com
#  ORGANIZATION: FABMation GmbH
#       CREATED: 01/29/2020 04:17:38 PM
#      REVISION:  001
#===============================================================================

set -o nounset                              # Treat unset variables as an error

echo "[i] Getting stats (this can take a long time!)"

num_files=$(ls -1 *.csv | wc -l)
raw_entries=$(cat *.csv | wc -l | sed ':a;s/\B[0-9]\{3\}\>/,&/;ta')
uniq_entries=$(cat *.csv | sort | uniq | wc -l | sed ':a;s/\B[0-9]\{3\}\>/,&/;ta')
single_entry=$(cat *.csv | sort | uniq -u | wc -l | sed ':a;s/\B[0-9]\{3\}\>/,&/;ta')
size=$(du -ch *.csv | grep total | awk '{ print $1 }')


echo "[i] Number of Files........: ${num_files}"
echo "[i] Total Entries..........: ${raw_entries}"
echo "[i] Total Entries (uniq)...: ${uniq_entries}"
echo "[i] Total Entries (1-time).: ${single_entry}"
echo "[i] Total Data size........: ${size}"