import csv

tsv_file = open("wikirank-fr-v2.tsv")
read_tsv = csv.reader(tsv_file, delimiter="\t")
articles = dict()
ln = 0
for row in read_tsv:
  ln = ln + 1
  # We skip strange lines (headers) that do not correspond to articles
  if row[0] == 'page_id' :
    print ('Line',ln,'is strange and looks like header')
    continue
  # skip lines with empty fields
  if row[0] == '':
    print ('Line',ln,'has empty id')
    continue
  if row[1] == '':
    print ('Line',ln,'has empty name')
    continue
  if row[2] == '':
    print ('Line',ln,'has empty quality')
    continue
  if row[3] == '':
    print ('Line',ln,'has empty popularity')
    continue
  if row[4] == '':
    print ('Line',ln,'has empty authors_interest')
    continue
  id = int(row[0])
  name = row[1]
  quality = float(row[2])
  pop = float(row[3])
  authors = float(row[4])
  if quality < 0 or quality > 100:
    print ('Line',ln,'has quality outside [0,100]')
    continue
  if id in articles:
    dejavu = articles[id]
    if dejavu == (id, name, quality, pop, authors):
#       print ('Doublon normal pour id = ', id)
       pass
    else:
      print ('Doublon étrange !!! pour id = ', id)
  else:
    articles[id] = (id, name, quality, pop, authors)

print ('Nombre d\'articles uniques ', len (articles))

print ()
print ('Trier par popularité et afficher le TOP20')
# Unix
# cat wikirank-fr-v2.tsv | sort | uniq | sort -t$'\t' -k4 -r -n | head -n20
# Attension la commande unix est simplifiée et ne tien pas compte les doublons étranges
srtd = sorted (articles.values(), key = lambda x : x[3], reverse = True)
for i in range (20):
  print (srtd[i])

print ()  
print ('TOP 20 des articles les moins populaires')
# Unix
# cat wikirank-fr-v2.tsv | sort | uniq | sort -t$'\t' -k4 -n | head -n20
# Attension la commande unix est simplifiée et ne tien pas compte les doublons étranges
for i in range (len(srtd) - 20, len(srtd)):
  print (srtd[i])

print ()  
print ("""Combien d'articles ont une popularité égale à zéro ?""")
# Unix
# cat wikirank-fr-v2.tsv | sort | uniq | grep -E $'.*\t.*\t.*\t0\t' | wc -l
# Attension la commande unix est simplifiée et ne tien pas compte les doublons étranges
c = 0
for id in articles:
  if articles[id][3] == 0: c = c + 1

print (c)

  
print ()
print ('''Visualisation. Il y a trop de points ~= 2m.
Il sera difficile de tous les dessiner, mais on peut essayer.
''')
import matplotlib as mpl
import matplotlib.pyplot as plt

popularity = [ line[3] for line in srtd]
authors_number = [ line[4] for line in srtd]

from numpy import corrcoef
c = corrcoef(popularity, authors_number)[0][1]

plt.scatter(popularity, authors_number )
plt.xlabel("Popularity")
plt.ylabel("# authors")
plt.title("Scatter plot. Pearson correlation = {}".format(c))
plt.show()

print('''ou bien 2d histogramme''')
plt.hist2d (popularity, authors_number, bins = [30, 30], norm = mpl.colors.LogNorm())
plt.colorbar()
plt.xlabel("Popularity")
plt.ylabel("# authors")
plt.title("Density plot with logarithmic colors. Pearson correlation = {}".format(c))
plt.show()


print('''ou en échelle logarithmique''')
plt.scatter(popularity, authors_number)
plt.xscale("log")
plt.yscale("log")
plt.xlabel("Popularity")
plt.ylabel("# authors")
plt.title("Scatter log-log plot. Pearson correlation = {}".format(c))
plt.show()


print("""Exemple de caractéristiques aberrantes. Articles très populaire mais peu d'auteurs""")
for a in srtd:
  if a[3] > 5000 and a[4] < 100:
    print (a)

print("""Articles peu populaire mais ayant trop d'auteurs""")
for a in srtd:
  if a[3] < 2000 and a[4] > 70:
    print (a)