git.lirion.de

Of git, get, and gud

aboutsummaryrefslogtreecommitdiffstats
path: root/refresh
blob: 22b9fd55e1ecbacfb92ecf0aa8b1918159c93f8e (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
# vim:syntax=python

"""
This script downloads data from holidata.net and writes it into files in the current directory.
Its output formats are parseable by timewarrior (https://taskwarrior.org/docs/timewarrior/).

This is very alpha. Since holidata.net has a few minor mistakes for Germany I'm writing this to
    test.xx-YY for now (I've just reported these, so allow the guys some time :) ). The files
    are still sourcable with that name :)

Also, for now this script only downloads de-DE. You will notice the complete lack of a main()
    definition, that is the next step: it will also carry parameters so you can download your
    own stuff.

Finally, I'm fixing the ISO-3166-2 violation for North Rhine Westphalia - holidata.net has de-NRW.
    That is the next bug to fix for them :)

tl;dr
SCRIPT IS NOT FINISHED, THIS IS A GIT - USE ONLY IF YOU UNDERSTAND AND IF YOU ARE FROM ZE GERMANY
"""

#import os, sys, re, time, argparse
import json, sys
reload(sys)
sys.setdefaultencoding('utf8')
from datetime import datetime
from time import sleep

if sys.version_info >= (3, 0):
    from urllib.request import urlopen
    from urllib.error import HTTPError
else:
    from urllib2 import urlopen, HTTPError

__hheader__ = "# File auto-generated by lirion.de's refresh,\n" \
            "# Date: " + datetime.strftime(datetime.now(), "%Y-%m-%d %H:%M:%S %z") + "\n" \
            "# data provided by holidata.net\n\n" \
            "define holidays:\n"

def headwrite(file):
    """
    This writes the default header to the file it is being passed on. Content see
    __hheader__ right above.
    """
    try:   
        open(file, 'w').close()
    except IOError:
        print("Cannot write to %s!", file)
        exit(2)
    with open(file, "a") as ofile:
        #f = open(ofile, "a")
        ofile.write(__hheader__)
 
def hfetch(locode, year):
    """
    This function downloads the holiday data for a given ISO-3166 code from holidata.net
    It will return a single string containing all of the JSON.
    """
    myurl = "https://holidata.net/" + mylocode + "/" + myyear + ".json"
    try:
        lines = urlopen(myurl).read().decode('utf-8')
    except HTTPError as httpe:
        if httpe.code == 404:
            print("holidata.net does not have data for %s(%s), for %d." % (mylocale, myregion,
                mydate))
        else:
            print(httpe.code, httpe.read())
    return lines

def hparse(lines, locode, year):
    """
    This function parses the JSON string we expect from holidata.net and writes its fragments to
    the specific region files, if a region is given. So, e.g.: a holiday for all of the US will
    not have a region definition and thus be written to file.en-US, whereas a holiday that is not
    valid for all of the US but is valid for Oregon will be written to file.en-OR.
    """
    islocdef = {}
    for line in lines.split('\n'):
        if line:
            sys.stdout.write(".")
            jdata = json.loads(line)
            mylocale = unicode(jdata['locale'])
            mycountry = mylocale.split('-')[0]
            myregion = unicode(jdata['region'])
            if myregion == "":
                myregion = mylocale.split('-')[1]
            # holidata.net violates ISO-3166-2:DE here:
            if myregion == "NRW":
                myregion = "NW" # or Neustadt/Weinstr., Palz > Pott
            myhlocale = unicode(mycountry + "-" + myregion)
            if jdata['date'] == "":
                print("%s has no date given for %s!", jdata['description'], jdata['locale'])
                mydate = unicode('1970_01_01')
            else:
                mydate = unicode(datetime.strftime(datetime.strptime(jdata['date'], '%Y-%m-%d'),
                    '%Y_%m_%d'))
            myfile = "test." + myhlocale
            if myhlocale not in islocdef:
                headwrite(myfile)
                with open(myfile, "a") as ofile:
                    #f = open(ofile, "a")
                    ofile.write("  " + mycountry + "-" + myregion + ":\n")
                islocdef[myhlocale] = myhlocale
            with open(myfile, "a") as ofile:
                ofile.write("    " + mydate + " = " + unicode(jdata['description']) + "\n")
            sleep(0.1)
            sys.stdout.flush()

if __name__ == "__main__":
    myyear = ""
    mylocode = ""
    if myyear == "":
        now = datetime.now()
        myyear = unicode(now.year)
    if mylocode == "":
        mylocode = "de-DE"
    sys.stdout.write("Fetching holiday data from holidata.net...")
    sys.stdout.flush()
    lines = hfetch(mylocode, myyear)
    print(" done.")
    if lines == "":
        print("No lines returned from holidata.net for %s!", mylocode)
        exit(3)
    sys.stdout.write("Parsing data")
    sys.stdout.flush()
    hparse(lines, mylocode, myyear)
    print(" done.")