#!/bin/bash
# this script is part of https://github.com/WolfgangFahl/ProceedingsTitleParser
# it gets the test data sets for
#
# 1. CEUR-WS
# 2. confref
# 3. crossref
# 4. dblp
# 5. wikicfp
#
# WF 2020-07-01
# WF 2026-09-19 --limit for crossref pages, connect timeout, confref cache first

#ansi colors
#http://www.csc.uvic.ca/~sae/seng265/fall04/tips/s265s047-tips/bash-using-colors.html
blue='\033[0;34m'
red='\033[0;31m'
green='\033[0;32m' # '\e[1;32m' is too bright for white bg.
endColor='\033[0m'

#
# a colored message
#   params:
#     1: l_color - the color of the message
#     2: l_msg - the message to display
#
color_msg() {
  local l_color="$1"
  local l_msg="$2"
  echo -e "${l_color}$l_msg${endColor}"
}

# error
#
#   show an error message and exit
#
#   params:
#     1: l_msg - the message to display
error() {
  local l_msg="$1"
  # use ansi red for error
  color_msg $red "Error: $l_msg" 1>&2
  exit 1
}

#
# show the usage
#
usage() {
  echo "usage: $0 [-h|--help][-l|--limit pages][--ceurws][--confref][--dblp][--wikicfp][-w|--wikidata][-g|--gnd]"
  echo "  -h|--help: show this usage"
  echo "  -l|--limit pages: maximum number of crossref pages of 1000 rows to download (default: all)"
  echo "  --ceurws: force getting samples for CEURWS and exit"
  echo "  --confref: get samples for confref and exit"
}

#
# download
#
download() {
  local l_src="$1"
  local l_target="$2"
  if [ ! -s $l_target ]
  then
    color_msg $blue "downloading $l_target from $l_src"
    curl -L --fail --connect-timeout 5 -o $l_target $l_src
    if [ $? -ne 0 ]
    then
      rm -f $l_target
      error "download of $l_src failed"
    fi
  else
    color_msg $green "$l_target already downloaded"
  fi
  # show number of lines
  # check if the file has json format
  if [[ $l_target =~ ^.*\.json$ ]]; then
    jq length $l_target
    # invalid json?
    if [ $? -ne 0 ]
    then
      color_msg $red "invalid json received for $l_target:"
      # show first few lines which might contain error message
      head $l_target
      color_msg $red "test will have to work-around this issue e.g. downloading a cached result"
    fi
  else
    wc -l $l_target
  fi
}

#
# get the data from CEUR-WS
#
getCEURWS() {
  color_msg $blue "getting sample data for CEUR-WS"
  samplehtml=$sampledir/ceurs-ws.html
  sampletxt=$sampledir/proceedings-ceur-ws.txt
  download https://ceur-ws.org/ $samplehtml
  if [ ! -f $sampletxt ]
  then
    color_msg $blue "filtering proceeding titles from $samplehtml into $sampletxt"
    cat $samplehtml  | python3 -c 'import html, sys; [print(html.unescape(l), end="") for l in sys.stdin]' | gawk '
BEGIN {
 ignorelines=1
 line=""
}
/MAINTABLE/ { ignorelines=0; next }
ignorelines { next }
# ignore
# get Vol from e.g.
# <a href="http://ceur-ws.org/Vol-2632/">MIning and REasoning with Legal texts.</a>
/ONLINE:.*<[Aa]\s+(HREF|href)=".*">.*<\/[aA]>/ {
  found=match($0,/[#/]Vol-[0-9]{1,4}/)
  if (found) {
     vol=substr($0, RSTART+1, RLENGTH-1)
     printf("|id=%s\n",vol)
     if (vol=="Vol-1")
       ignorelines=1
  }
  next
}
/<[Aa]\s+(HREF|href)=".*">/,/<\/[aA]>/ { next }
/ftp\/pub/ { next }
/ARCHIVE:/ {next }
/Published on CEUR-WS:/ { next }
/<[aA]\sname/ { next }
/Edited by: / { next }
/<font>/ { next }
/<font size=.?-2.?>/ { next }
/<\/font>/ { next }
/<TD\salign="left"\sbgcolor="#DCDBD7">/ { next }
/#00000/ { next }
/#CCEBC7/ { next }
/#F0D2F5/ { next }
/<\/TD>/ { next }
/<\/td>/ { next }
/<[\/]?table|TABLE>/ { next }
/<[\/]?tr|TR>/ { next }
/--/ { print ""; next }
# FILTER raw html
{
  line=line$0
  gsub("<BR>","",line)
  gsub("<br>","",line)
  gsub("<sup>2</sup>","2",line)
  gsub("<(td|TD).bgcolor=.#FFFFFF.>","",line)
  # nbsp
  gsub(" </TD>","",line)
  # trim
  gsub(/^[ \t]+/, "", line);
  gsub(/[ \t]+$/, "", line);
  # fix "5 th" typo from http://ceur-ws.org/Vol-2341"
  gsub("5 th","5th",line)
  if (index(line,"Submitted by") >0){
    printf ("%s",line)
    line=""
  }
}' | sed  '/^$/d'  > $sampletxt
 else
    color_msg $green "$sampletxt already filtered from $samplehtml"
 fi
 wc -l $sampletxt
}

#
# get dblp json file
#
getDblp() {
  source=dblp.json
  download https://wiki.bitplan.com/images/confident/$source $sampledir/$source
}

#
# get wikiCFP crawled json files
#
getWikiCFP() {
  source=wikicfp.tgz
  download https://wiki.bitplan.com/images/confident/$source $sampledir/$source
  pwd=$(pwd)
  cd $sampledir
  # unpack
  tar xvfz $source
  cd $pwd
}

#
# get the text files
#
getText() {
  for source in dblp wikidata
  do
     target=proceedings-$source.txt
     download https://wiki.bitplan.com/images/confident/$target $sampledir/$target
  done
}

#
# download from crossref RESTful API via cursor
#
downloadWithCursor() {
  local l_rows="$1"
  local l_index="$2"
  local l_cursor="$3"
  target=$sampledir/crossref-$l_index.json
  src="https://api.crossref.org/types/proceedings/works?select=event,title,DOI&rows=$l_rows&cursor=$l_cursor"
  download $src $target
}

#
# get Crossref data
# see also https://github.com/TIBHannover/confIDent-dataScraping
#
getCrossRef() {
  rows=1000
  index=1
  totalRows=0
  # force while entry
  total=$rows
  downloadWithCursor $rows $index "*"
  while [ $totalRows -lt $total ]
  do
    if [ "$limit" != "" ] && [ $index -ge $limit ]
    then
      color_msg $blue "crossref page limit $limit reached"
      break
    fi
    target=$sampledir/crossref-$index.json
    status=$(jq '.status' $target | tr -d '"')
    total=$(jq '.message["total-results"]' $target)
    # get and remove quotes from cursor
    cursor=$(jq '.message["next-cursor"]' $target | tr -d '"' | python -c "import urllib.parse;print (urllib.parse.quote(input()))"
    target=$sampledir/crossref-$l_index.json)
    startindex=$(jq '.message.query["start-index"]' $target)
    perpage=$(jq '.message["items-per-page"]' $target)
    index=$[$index+1]
    if [ "$status" == "ok" ]
    then
      totalRows=$[$totalRows+$rows]
    else
      # force while exit
      totalRows=1
      total=0
      # remove invalid
      mv $target $target.err
    fi
    echo "status: $status index: $index $totalRows of $total startindex: $startindex perpage=$perpage cursor:$cursor"
    if [ $totalRows -lt $total ]
    then
      # wait a bit
      sleep 2
      downloadWithCursor $rows $index "$cursor"
    fi
  done
  cat $sampledir/crossref-*.json | jq .message.items[].title | cut -f2 -d'[' | cut -f2 -d'"' | grep -v "]" | tr -s '\n' > $sampledir/proceedings-crossref.txt
}

#
# get sample data from confref
# portal.confref.org is gone (2026) so the cached copies are used
#
getConfRef() {
  for entity in countries areas conferences
  do
    target=$sampledir/confref-$entity.json
    src="https://wiki.bitplan.com/images/confident/confref-$entity.json"
    download $src $target
  done
}

#
# get a cached SQLLite3 database from the default source
#
getCachedDatabase() {
  local l_db="$1"
	src=https://wiki.bitplan.com/images/confident/$l_db
  target=$cachedir/$l_db
	download $src $target
}
#
# get the cached wiki data queries for country and province
#
getWikiDataRegion() {
	for r in Country Province City
	do
    db=${r}_wikidata.db
		getCachedDatabase $db
	done
}

# get the GND SQL database
getGND() {
  getCachedDatabase Event_gnd.db
}

#
# get samples from all relevant sources
#
getAll() {
  getConfRef
  getCrossRef
  getCEURWS
  getDblp
  getWikiCFP
  getText
  getGND
  getWikiDataRegion
}

# commandline option
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
#echo $DIR
sampledir="$DIR/../sampledata"
cachedir="$DIR/../cache"

for d in $sampledir $cacheDir
do
	if [ ! -d $d ]
	then
	  color_msg $blue  "creating $d directory"
	  mkdir -p $d
	else
	  color_msg $green "$d directory already exists"
	fi
done

limit=""
while [  "$1" != ""  ]
do
  option=$1
  shift
  case $option in
    -h|--help)
      usage
      exit 0;;
    -l|--limit)
      limit=$1
      shift
      ;;
    --confref)
      getConfRef
      exit 0;;
    --ceurws)
      html=$sampledir/ceurs-ws.html
      txt=$sampledir/proceedings-ceur-ws.txt
      color_msg $blue "forcing getsamples for CEUR-WS by deleting $html and $txt"
      rm $html $txt
      getCEURWS
      exit 0;;
    --dblp)
      getDblp
      exit 0;;
    --wikicfp)
      getWikiCFP
      exit 0;;
    -w|--wikidata)
      getWikiDataRegion
      exit 0;;
    -g|--gnd)
      getGND
      exit 0;;
  esac
done

getAll
