"""
balablasb
"""
from wsgi.models import *
from sqlalchemy.sql.expression import text
from wsgi import db
import numpy as np
from typing import Dict, List
from collections import OrderedDict
from wsgi.util import colorize, yellow, green, red
from tabulate import tabulate
import sys
from itertools import product
import redis
from wsgi.config import NPQA_REDIS_DB, NPQA_REDIS_HOST, NPQA_REDIS_PORT, NPQA_REDIS_WORK_DB, HR_ID
import pickle
from abc import ABC
from itertools import product
import json
import yaml
from logging import getLogger
logger = getLogger(__name__)
NP_NUM_COLS = 24
index_gkz = None # type: OrderedDict
gkz_index = None # type: dict
#: Enhält die Gemeindeergebnisse aller betroffenen Wahlen. Wird durch :py:func:`np_complete_erg_array_for_wahl`
#: befüllt, das wiederum in :py:func:`initialize` aufgerufen wird.
#: type: Dict[wahl_id, GemeindeErgebnis]
complete_erg_arrays = None # type: Dict[int, np.array]
indizes_akt_aus = None # type: set
redis_connection = None
redis_work_connection = None
KRIT_IDS_OF_KRITERIEN_WITH_KRITKLASSEN = None # type: set
DEFAULT_REDIS_NPQAKRIT_KEYS = None # type: set
REDIS_NPQAKRIT_PREFIX = 'npqakrit'
REDIS_NPQAKRIT_KEY_LIFETIME_SECONDS = 1800 # 30 Minuten
class KriteriumWithoutKlassenException(Exception):
pass
def _thou(n :int):
"""
Eine Zahl mit Tausenderpunkten als String zurückliefern
:param n: Eine Integer Zahl
:return: String, Zahl mit Tausenderpunkten
"""
r = []
for i, c in enumerate(reversed(str(n))):
if i and (not (i % 3)):
r.insert(0, '.')
r.insert(0, c)
return ''.join(r)
[Doku]def get_redis_npqakrit_key(krit_id :int, wahl_alt_id :int, wahl_akt_id :int, bl :int):
"""
Erstelle aus den Parametern einen eindeutigen Key, indem das REDIS_NPQAKRIT_PREFIX und die
Zahlen mit ":" gejoined werden.
>>> get_redis_npqakrit_key(132, 16, 18, 0)
>>> 'npqakrit:132:16:18:0'
:param krit_id:
:param wahl_alt_id:
:param wahl_akt_id:
:param bl:
:return: str
"""
return ":".join(map(str, [REDIS_NPQAKRIT_PREFIX, krit_id, wahl_alt_id, wahl_akt_id, bl]))
[Doku]def add_key_to_redis_keys_for_qa(key):
"""
Setze den angegebenen Key mit einer value=key und einer Expirytime von REDIS_NPQAKRIT_KEY_LIFETIME_SECONDS
Der Key ist ein Key von get_redis_npqakrit_key(), zB. npqakrit:132:18:16:0
Alle "npqakrit*" Keys werden bei der nächsten QA brücksichtigt und damit stehen sie dann im Cache bereit.
:param key:
:return:
"""
redis_connection.set(key, key, ex=REDIS_NPQAKRIT_KEY_LIFETIME_SECONDS)
[Doku]def initialize(force_reload_ergebnisse = False):
"""
Initialisiere alle notwendigen Datenstrukturen für die Qa:
* redis_connection
* redis_work_connection - die RedisDB, in der die QA berechnet wird, anschliessend werden alle Objekte von dort mittels
mset() atomic in die richtie RedisDB geschrieben. Damit gibts immer einen konsistenten Stand.
* DEFAULT_REDIS_NPQAKRIT_KEYS - Alle NpQaKritKeys (npqakrit:132:18:16:0) die automatisch bei jeder QA berechnet werden.
Default mäßig ist das eine Kombination von alle Kriterien der Hochrechnung und alte Wahl/fiktive Basis gegen neue Wahl
* KRIT_IDS_OF_KRITERIEN_WITH_KRITKLASSEN - IDs der Kriterien, deren Klassen wiederum aus Kriterien aka KritKlassen bestehen.
Wird von NpQaKrit genutzt um entweder eine NpQaKlasse oder eine NpQaKritKlasse zu erzeugen
* complete_erg_arrays - Dictionary mit key=wahl_id, value=np_complete_erg_array_for_wahl, sprich alle Gemeindeergebnisse der
angegebenen Wahl in einem NumPy Array.
* index_gkz - Dict von gemeinde[gkz]-> Index im complete_erg_array
* gkz_index - umgekehrt
:param force_reload_ergebnisse:
:return:
"""
hr = Hochrechnung.query.get(HR_ID) # type: Hochrechnung
logger.info("Initalize npqa for Hochrechnung={} ({}) and force_reload_ergebnisse={}".format(str(hr), hr.id, force_reload_ergebnisse))
global index_gkz, gkz_index, complete_erg_arrays, indizes_akt_aus, redis_connection, \
redis_work_connection, KRIT_IDS_OF_KRITERIEN_WITH_KRITKLASSEN, DEFAULT_REDIS_NPQAKRIT_KEYS
if redis_connection is None:
redis_connection = redis.Redis(host=NPQA_REDIS_HOST, port=NPQA_REDIS_PORT, db=NPQA_REDIS_DB)
if redis_work_connection is None:
redis_work_connection = redis.Redis(host=NPQA_REDIS_HOST, port=NPQA_REDIS_PORT, db=NPQA_REDIS_WORK_DB)
if DEFAULT_REDIS_NPQAKRIT_KEYS is None:
kriterien = [krit.id for krit in hr.kriterien]
wahlen_alt = [hr.wahl_alt.id, hr.fiktive_basis_id]
wahlen_akt = [hr.wahl_akt.id]
combinations = list(product(kriterien, wahlen_alt, wahlen_akt, [hr.bl]))
DEFAULT_REDIS_NPQAKRIT_KEYS = set([get_redis_npqakrit_key(*combination) for combination in combinations if combination[1] != combination[2]])
logger.info("Number of Default Kriterien/Wahlen/Bl combinations: {}".format(len(DEFAULT_REDIS_NPQAKRIT_KEYS)))
if KRIT_IDS_OF_KRITERIEN_WITH_KRITKLASSEN is None:
KRIT_IDS_OF_KRITERIEN_WITH_KRITKLASSEN = set([k.krit_id for k in KritKlasse.query.all()])
if complete_erg_arrays is not None and not force_reload_ergebnisse:
logger.info("complete_erg_arrays are filled and not force_reload_ergebnisse. Return without initialization")
return
redis_connection.flushdb()
redis_work_connection.flushdb()
complete_erg_arrays = {}
index_gkz = OrderedDict([(i, e[0]) for i, e in enumerate(db.engine.execute("select gkz from gemeinden order by gkz asc").fetchall())])
gkz_index = dict([(gkz, index) for index, gkz in index_gkz.items()])
wahl_ids = [hr.wahl_akt_id, hr.wahl_alt_id, hr.fiktive_basis_id]
for wahl_id in wahl_ids:
logger.info("Get Complete Erg Array for wahl_id={}".format(wahl_id))
complete_erg_arrays[wahl_id] = np_complete_erg_array_for_wahl(wahl_id, hr.bl)
logger.info("npqa data initialized")
def clear_cache():
global redis_connection
if redis_connection is None:
redis_connection = redis.Redis(host=NPQA_REDIS_HOST, port=NPQA_REDIS_PORT, db=NPQA_REDIS_DB)
redis_connection.flushdb()
[Doku]def np_complete_erg_array_for_wahl(wahl_id: int, bl :int = 0) -> np.array:
"""
return a numpy array mit gkz, wab, abg, gig, p01 ... p20 für alle Gemeinden der Wahl. *Keine Fake Ergebnisse*,
sortiert nach gkz.
:param wahl_id: wahl_it
:param bl: Bundesland
:return: np.array vom typ int32)
"""
# <editor-fold desc="Code">
global gemeinden_master_array
sql = """
select
g.gkz,
coalesce(e.wab, 0),
coalesce(e.abg, 0),
coalesce(e.gig, 0),
coalesce(e.p01, 0),
coalesce(e.p02, 0),
coalesce(e.p03, 0),
coalesce(e.p04, 0),
coalesce(e.p05, 0),
coalesce(e.p06, 0),
coalesce(e.p07, 0),
coalesce(e.p08, 0),
coalesce(e.p09, 0),
coalesce(e.p10, 0),
coalesce(e.p11, 0),
coalesce(e.p12, 0),
coalesce(e.p13, 0),
coalesce(e.p14, 0),
coalesce(e.p15, 0),
coalesce(e.p16, 0),
coalesce(e.p17, 0),
coalesce(e.p18, 0),
coalesce(e.p19, 0),
coalesce(e.p20, 0)
from gemeinden g left outer join gemeindeergebnisse e on g.gkz=e.gkz
and e.is_fake = FALSE
and e.wahl_id=:wahl_id """
if bl != 0:
sql += " where g.gkz between :bl * 10000 and :bl * 10000 + 9999 "
sql += "order by gkz asc"
sql = text(sql)
result = db.engine.execute(sql, wahl_id=wahl_id, bl=bl).fetchall()
return np.array(result, dtype=np.int32)
# </editor-fold>
[Doku]def np_erg_array_for_gks(a: np.array, gkzs: list) -> np.array:
"""
Return the numpy erg array with only the gkzs from the list
:param a:
:param gkzs:
:return:
"""
# <editor-fold desc="Code">
indizes = [gkz_index[gkz] for gkz in gkzs]
return np.take(a, indizes, axis=0)
# </editor-fold>
[Doku]def sum_np_erg_array(a: np.array) -> np.array:
"""
Summiere alle Spalten des Arrays
:param a:
:return:
"""
return np.sum(a, axis=0)
iwab = 0
iabg = 1
igig = 2
class NpBaseKlasse(ABC):
def __init__(self):
self.wab_aus = 0 # type: int
self.wab_ges = 0 # type: int
self.gem_aus = 0 # type: int
self.krit_id = 0 # type: int
self.klasse_id = 0 # type: int
self.wahl_alt_id = 0 # type: int
self.wahl_akt_id = 0 # type: int
self.hr = None # type: Hochrechnung
self.has_ergebnisse = False # type: bool
self.np_alt_ges = None # type: np.ndarray
self.np_alt_ges_p = None # type: np.ndarray
self.np_alt_aus = None # type: np.ndarray
self.np_alt_aus_p = None # type: np.ndarray
self.np_akt_aus = None # type: np.ndarray
self.np_akt_aus_p = None # type: np.ndarray
self.np_akt_ges = None # type: np.ndarray
self.np_akt_ges_p = None # type: np.ndarray
self.np_neu_trend = None # type: np.ndarray
self.np_neu_trend_p = None # type: np.ndarray
self.np_diff_p = None # type: np.ndarray
self.np_diff = None # type: np.ndarray
def prozent_array(self, a: np.ndarray):
p = np.zeros_like(a, dtype=np.float) # type: np.ndarray
if a[1] == 0: # abg == 0
return p
p[0] = self.wab_aus / self.wab_ges if self.wab_ges > 0 else 0.0 # wab
p[1] = a[1] / a[0] if a[0] > 0 else 0.0 # abg
p[2] = a[2] / a[1] # gig
np.divide(a[3:], a[2], p[3:])
return p
def berechne_neu_trend(self):
# Wenn noch keine Gemeinden ausgezählt sind, kann noch kein Trend für die Klasse genommen werden
# In NpQaKlasse und NpQaKritKlasse werden np_neu_trend_p und np_neu_trend vor dem Aufruf von
# berechne_neu_trend ausgenullt. Dieser Werte bleiben einfach bestehen
if self.gem_aus == 0:
return
self.np_neu_trend_p = self.np_alt_ges_p + self.np_diff_p
self.np_neu_trend[0] = self.np_akt_ges[0] # WAB von akt_ges
self.np_neu_trend[1] = round(self.np_neu_trend[0] * self.np_neu_trend_p[1]) # ABG
self.np_neu_trend[2] = round(self.np_neu_trend[1] * self.np_neu_trend_p[2]) # GIG
__t = np.rint(np.multiply(self.np_neu_trend_p[3:], self.np_neu_trend[2])).astype(np.int64)
self.np_neu_trend[3:] = __t
self.np_neu_trend[2] = np.sum(__t)
if self.np_neu_trend[1] < self.np_neu_trend[2]: self.np_neu_trend[1] = self.np_neu_trend[2] # Rundungsfehler ausgleichen
if self.np_neu_trend[0] < self.np_neu_trend[1]: self.np_neu_trend[0] = self.np_neu_trend[1] # Rundungsfehler ausgleichen
def disguise_as_old_qa(self):
self.has_ergebnisse = self.gem_aus > 0
self.hr = Hochrechnung.query.get(HR_ID)
def incorporate_fake_ergebnisse(self):
"""
Query the database if any Fake Ergebnisse are set for this class and adapt the self.np_sum* accordingly
"""
pass
def before_pickle(self):
self.klasse = None
self.krit = None
self.wahl_alt = None
self.wahl_akt = None
def after_pickle(self):
self.krit = Kriterium.query.get(self.krit_id)
self.klasse = Klasse.query.get((self.krit.id, self.klasse_id))
self.wahl_alt = BaseWahl.query.get(self.wahl_alt_id)
self.wahl_akt = BaseWahl.query.get(self.wahl_akt_id)
def as_dict(self):
keys_for_dict= ('wahl_akt_id', 'wahl_alt_id', 'krit_id', 'klasse_id', 'bl', 'is_fake', 'klasse_gkzs', \
'np_alt_ges', 'np_alt_aus', 'np_akt_aus', 'np_akt_ges', 'np_diff' \
'wab_aus', 'wab_ges', 'gem_aus', 'gem_ges')
d = {}
for key in keys_for_dict:
try:
item = getattr(self, key)
except AttributeError:
continue
if isinstance(item, (np.int64, np.int32)):
item = int(item)
if isinstance(item, np.ndarray):
item = item.tolist()
d[key] = item
return d
@property
def asciitable(self):
headers = ['','gkz', 'wab', 'abg', 'gig', 'p01', 'p02', 'p03', 'p04', 'p05', 'p06', 'p07', 'p08', 'p09', 'p10', 'p11', 'p12', 'p13', 'p14', 'p15', 'p16', 'p17', 'p18', 'p19', 'p20']
l = []
l.append(['alt_ges'] + [_thou(value) for value in self.np_alt_ges.tolist()])
l.append(['alt_aus'] + [_thou(value) for value in self.np_alt_aus.tolist()])
l.append(['akt_ges'] + [_thou(value) for value in self.np_akt_ges.tolist()])
l.append(['akt_aus'] + [_thou(value) for value in self.np_akt_aus.tolist()])
l.append(['diff'] + [_thou(value) for value in self.np_diff.tolist()])
return tabulate(l, headers=headers)
[Doku]class NpQaKlasse(NpBaseKlasse):
"""
Querschnittsanalyse über eine einzige Klasse. Summiert die Ergebnisse der Klasse selbst.
"""
def __init__(self, wahl_akt_id: int, wahl_alt_id: int, krit_id: int, klasse_id: int, bl: int = 0):
super().__init__() # Superklasse ist nur ein Container, damit die Methoden der Klasse auch für KritKlassen verwendet werden können
self.wahl_akt_id = wahl_akt_id
self.wahl_alt_id = wahl_alt_id
self.krit_id = krit_id
self.klasse_id = klasse_id
self.wahl_akt = None
self.wahl_alt = None
self.bl = bl
self.is_fake = False
sql = "select gkz from codes where krit_id=:krit_id and klasse_id=:klasse_id"
if self.bl != 0:
sql += " and gkz between :bl * 10000 and :bl * 10000 + 9999 "
self.klasse_gkzs = [t[0] for t in db.engine.execute(text(sql), krit_id=self.krit_id, klasse_id=self.klasse_id, bl=bl).fetchall()]
self._klasse_indizes = [gkz_index[gkz] for gkz in self.klasse_gkzs]
self._np_erg_wahl_akt = complete_erg_arrays[self.wahl_akt_id]
self._np_erg_wahl_alt = complete_erg_arrays[self.wahl_alt_id]
self._indizes_abg = self._np_erg_wahl_akt[:, 2].nonzero()[0].tolist() # abg != 0
self._indizes_abg = list(set(self._klasse_indizes).intersection(set(self._indizes_abg)))
# GKZ, wab, abg, gig, .....
self.np_ergebnisse_alt_ges = self._np_erg_wahl_alt.take(self._klasse_indizes, axis=0) # Alle Ergebnisse in AltGes
self.np_ergebnisse_alt_aus = self._np_erg_wahl_alt.take(self._indizes_abg, axis=0) # Nur abgegebene in AltAus
self.np_ergebnisse_akt_ges = self._np_erg_wahl_akt.take(self._klasse_indizes, axis=0) # Alle Ergebnisse in AltGes - vor allem ALLE WAB
self.np_ergebnisse_akt_aus = self._np_erg_wahl_akt.take(self._indizes_abg, axis=0) # Nur abgegeben in AktAus
# Die unnötigen GKZ Daten werden weggeschnitten
self.np_alt_ges = self.np_ergebnisse_alt_ges.sum(axis=0)[1:] # type: np.ndarray
self.np_alt_aus = self.np_ergebnisse_alt_aus.sum(axis=0)[1:] # type: np.ndarray
self.np_akt_ges = self.np_ergebnisse_akt_ges.sum(axis=0)[1:] # type: np.ndarray
self.np_akt_aus = self.np_ergebnisse_akt_aus.sum(axis=0)[1:] # type: np.ndarray
self.incorporate_fake_ergebnisse()
self.wab_aus = int(self.np_akt_aus[0])
self.wab_ges = int(self.np_akt_ges[0])
self.gem_aus = len(self._indizes_abg)
self.gem_ges = len(self.klasse_gkzs)
self.np_alt_ges_p = self.prozent_array(self.np_alt_ges) # type: np.ndarray
self.np_alt_aus_p = self.prozent_array(self.np_alt_aus) # type: np.ndarray
self.np_akt_ges_p = self.prozent_array(self.np_akt_ges) # type: np.ndarray
self.np_akt_aus_p = self.prozent_array(self.np_akt_aus) # type: np.ndarray
self.np_diff = self.np_akt_aus - self.np_alt_aus # type: np.ndarray
self.np_diff_p = self.np_akt_aus_p - self.np_alt_aus_p # type: np.ndarray
self.np_hq_p = np.nan_to_num(np.divide(self.np_akt_aus_p, self.np_alt_aus_p)) # type: np.ndarray
self.np_neu_trend_p = np.zeros_like(self.np_diff_p, dtype=np.float) # type: np.ndarray
self.np_neu_trend = np.zeros_like(self.np_diff, dtype=np.int64) # type: np.ndarray
self.berechne_neu_trend()
self.disguise_as_old_qa()
class NpQaKritKlasse(NpBaseKlasse):
def __init__(self, wahl_akt_id: int, wahl_alt_id: int, krit_id: int, klasse_id: int, bl: int = 0):
super().__init__() # Superklasse ist nur ein Container, damit die Methoden der Klasse auch für KritKlassen verwendet werden können
self.wahl_akt_id = wahl_akt_id
self.wahl_alt_id = wahl_alt_id
self.krit_id = krit_id
self.klasse_id = klasse_id
self.wahl_akt = None
self.wahl_alt = None
self.bl = bl
self.kritklasse = KritKlasse.query.filter_by(krit_id=krit_id, klasse_id=klasse_id, bl=bl).one()
self.result_krit_id = self.kritklasse.result_krit_id
self.krit_used_as_klasse = NpQaKrit(self.result_krit_id, wahl_alt_id, wahl_akt_id, bl) # type: NpQaKrit
# KRIT_USED_AS_KLASSE auslesen und Daten wie eine Klasse aufbereiten
self.np_ergebnisse_alt_ges = self.krit_used_as_klasse.np_ergebnisse_alt_ges.copy()
self.np_ergebnisse_alt_aus = self.krit_used_as_klasse.np_ergebnisse_alt_aus.copy()
self.np_ergebnisse_akt_ges = self.krit_used_as_klasse.np_ergebnisse_akt_ges.copy()
self.np_ergebnisse_akt_aus = self.krit_used_as_klasse.np_ergebnisse_akt_aus.copy()
self.np_alt_ges = self.krit_used_as_klasse.np_sum_alt_ges.copy()
self.np_alt_aus = self.krit_used_as_klasse.np_sum_alt_aus.copy()
self.np_akt_ges = self.krit_used_as_klasse.np_sum_akt_ges.copy()
self.np_akt_aus = self.krit_used_as_klasse.np_sum_akt_aus.copy()
self.np_neu_trend = self.krit_used_as_klasse.np_sum_neu_trend.copy()
self.np_diff = self.krit_used_as_klasse.np_sum_diff.copy()
self.np_alt_ges_p = self.krit_used_as_klasse.np_sum_alt_ges_p.copy()
self.np_alt_aus_p = self.krit_used_as_klasse.np_sum_alt_aus_p.copy()
self.np_akt_ges_p = self.krit_used_as_klasse.np_sum_akt_ges_p.copy()
self.np_akt_aus_p = self.krit_used_as_klasse.np_sum_akt_aus_p.copy()
self.np_neu_trend_p = self.krit_used_as_klasse.np_sum_neu_trend_p.copy()
self.np_diff_p = self.krit_used_as_klasse.np_sum_diff_p.copy()
self.wab_aus = self.krit_used_as_klasse.wab_aus
self.wab_ges = self.krit_used_as_klasse.wab_ges
self.gem_aus = self.krit_used_as_klasse.gem_aus
self.gem_ges = self.krit_used_as_klasse.gem_ges
self.is_fake = self.krit_used_as_klasse.is_fake
# Daten des krit_used_as_klasse übernommen, nun als NpQaKlasse darstellen
#
# Prozent, Diff, NeuTrend und Haltequoten berechnen, wie es die NpQaKlasse auch machen
#
self.np_hq_p = np.nan_to_num(np.divide(self.np_akt_aus_p, self.np_alt_aus_p)) # type: np.ndarray
self.disguise_as_old_qa()
class NpQaKrit:
def __init__(self, krit_id: int, wahl_akt_id: int, wahl_alt_id: int, bl: int = 0):
self.wahl_akt_id = wahl_akt_id
self.wahl_alt_id = wahl_alt_id
self.bl = bl
self.wahl_alt = None
self.wahl_akt = None
self.krit = None
self.krit_id = krit_id
self.redis_key = get_redis_npqakrit_key(krit_id=krit_id, wahl_alt_id=wahl_alt_id, wahl_akt_id=wahl_akt_id, bl=bl)
sql = text("select klasse_id from BaseKlassen where krit_id=:krit_id order by klasse_id asc")
klasse_ids = [t[0] for t in db.engine.execute(sql, krit_id=self.krit_id).fetchall()]
if len(klasse_ids) == 0:
__krit = Kriterium.query.get(self.krit_id)
raise KriteriumWithoutKlassenException("Das Kriterium {} - {} hat keine Klassen".format(self.krit_id, __krit.bez))
self.qa_klassen = [] # type: List[NpQaKlasse]
for klasse_id in klasse_ids: # type: Klasse
NpQaKlassenType = NpQaKritKlasse if self.krit_id in KRIT_IDS_OF_KRITERIEN_WITH_KRITKLASSEN else NpQaKlasse
self.qa_klassen.append(NpQaKlassenType(wahl_akt_id=wahl_akt_id, wahl_alt_id=wahl_alt_id, krit_id=self.krit_id, klasse_id=klasse_id, bl=self.bl))
self.anz_klassen = len(self.qa_klassen)
self.is_fake = True in [k.is_fake for k in self.qa_klassen]
self.np_ergebnisse_alt_ges = np.concatenate([np_klasse.np_ergebnisse_alt_ges for np_klasse in self.qa_klassen])
self.np_ergebnisse_alt_aus = np.concatenate([np_klasse.np_ergebnisse_alt_aus for np_klasse in self.qa_klassen])
self.np_ergebnisse_akt_ges = np.concatenate([np_klasse.np_ergebnisse_akt_ges for np_klasse in self.qa_klassen])
self.np_ergebnisse_akt_aus = np.concatenate([np_klasse.np_ergebnisse_akt_aus for np_klasse in self.qa_klassen])
self.np_klassen_alt_ges = np.array([np_klasse.np_alt_ges for np_klasse in self.qa_klassen]) # type: np.ndarray
self.np_klassen_alt_aus = np.array([np_klasse.np_alt_aus for np_klasse in self.qa_klassen]) # type: np.ndarray
self.np_klassen_akt_ges = np.array([np_klasse.np_akt_ges for np_klasse in self.qa_klassen]) # type: np.ndarray
self.np_klassen_akt_aus = np.array([np_klasse.np_akt_aus for np_klasse in self.qa_klassen]) # type: np.ndarray
self.np_klassen_diff = np.array([np_klasse.np_diff for np_klasse in self.qa_klassen]) # type: np.ndarray
self.np_klassen_neu_trend = np.array([np_klasse.np_neu_trend for np_klasse in self.qa_klassen]) # type: np.ndarray
self.np_klassen_alt_ges_p = np.array([np_klasse.np_alt_ges_p for np_klasse in self.qa_klassen]) # type: np.ndarray
self.np_klassen_alt_aus_p = np.array([np_klasse.np_alt_aus_p for np_klasse in self.qa_klassen]) # type: np.ndarray
self.np_klassen_akt_ges_p = np.array([np_klasse.np_akt_ges_p for np_klasse in self.qa_klassen]) # type: np.ndarray
self.np_klassen_akt_aus_p = np.array([np_klasse.np_akt_aus_p for np_klasse in self.qa_klassen]) # type: np.ndarray
self.np_klassen_diff_p = np.array([np_klasse.np_diff_p for np_klasse in self.qa_klassen]) # type: np.ndarray
self.np_klassen_neu_trend_p = np.array([np_klasse.np_neu_trend_p for np_klasse in self.qa_klassen]) # type: np.ndarray
self.np_sum_alt_ges = self.np_klassen_alt_ges.sum(axis=0) # type: np.ndarray
self.np_sum_alt_aus = self.np_klassen_alt_aus.sum(axis=0) # type: np.ndarray
self.np_sum_akt_aus = self.np_klassen_akt_aus.sum(axis=0) # type: np.ndarray
self.np_sum_akt_ges = self.np_klassen_akt_ges.sum(axis=0) # type: np.ndarray
self.np_sum_diff = self.np_sum_akt_aus - self.np_sum_alt_aus # type: np.ndarray
self.np_sum_neu_trend = self.np_klassen_neu_trend.sum(axis=0) # type: np.ndarray
self.gem_aus = int(sum([npklasse.gem_aus for npklasse in self.qa_klassen])) # type: NpQaKlasse
self.gem_ges = int(sum([npklasse.gem_ges for npklasse in self.qa_klassen])) # type: NpQaKlasse
self.wab_aus = int(sum([npklasse.wab_aus for npklasse in self.qa_klassen])) # type: NpQaKlasse
self.wab_ges = int(sum([npklasse.wab_ges for npklasse in self.qa_klassen])) # type: NpQaKlasse
self.np_sum_alt_ges_p = self.prozent_array(self.np_sum_alt_ges) # type: np.ndarray
self.np_sum_alt_aus_p = self.prozent_array(self.np_sum_alt_aus) # type: np.ndarray
self.np_sum_akt_ges_p = self.prozent_array(self.np_sum_akt_ges) # type: np.ndarray
self.np_sum_akt_aus_p = self.prozent_array(self.np_sum_akt_aus) # type: np.ndarray
self.np_sum_diff_p = self.np_sum_akt_aus_p - self.np_sum_alt_aus_p # type: np.ndarray
self.np_sum_neu_trend_p = self.prozent_array(self.np_sum_neu_trend) # type: np.ndarray
self.np_sum_hq_p = np.nan_to_num(np.divide(self.np_sum_akt_aus_p, self.np_sum_alt_aus_p)) # type: np.ndarray
self.disguise_as_old_qa()
def prozent_array(self, a: np.ndarray):
p = np.zeros_like(a, dtype=np.float) # type: np.ndarray
if a[1] == 0: # abg == 0
return p
p[0] = self.wab_aus / self.wab_ges if self.wab_ges > 0 else 0.0 # wab
p[1] = a[1] / a[0] if a[0] > 0 else 0.0 # abg
p[2] = a[2] / a[1] # gig
np.divide(a[3:], a[2], p[3:])
return p
def berechne_ungewichtet(self):
self.np_ungewichtet_p = self.np_sum_alt_ges_p + self.np_diff_p
self.np_ungewichtet = np.zeros_like(self.np_sum_alt_ges)
def disguise_as_old_qa(self):
from wsgi.qa import SimpleQaErgebnis, QaSimpleErgebnisAltAus, QaSimpleErgebnisAltGes, QaSimpleErgebnisAktAus, QaSimpleErgebnisNeuTrend, QaSimpleErgebnisVergleich
# 0 1 2 3 4
# gkz, wab, abg, gig, p01
self.has_ergebnisse = self.np_sum_akt_aus[iabg] > 0
self.hr = Hochrechnung.query.get(HR_ID)
self.qaklassen_alt_aus = []
self.qaklassen_akt_aus = []
self.qaklassen_alt_ges = []
self.qaklassen_neu_trend = []
self.qaklassen_vergleich = []
if self.krit is None: self.krit = Kriterium.query.get(self.krit_id)
for i, klasse in enumerate(self.krit.klassen):
qa_klasse = self.qa_klassen[i] # type: NpQaKlasse
params = lambda np_klasse: (self.krit_id, klasse.klasse_id, klasse.bez, [None, qa_klasse.gem_aus] + [int(j) for j in np_klasse.tolist()], qa_klasse.gem_ges, qa_klasse.wab_ges)
alt_aus = QaSimpleErgebnisAltAus.from_erg_row(*params(self.np_klassen_alt_aus[i]))
akt_aus = QaSimpleErgebnisAktAus.from_erg_row(*params(self.np_klassen_akt_aus[i]))
alt_ges = QaSimpleErgebnisAltGes.from_erg_row(*params(self.np_klassen_alt_ges[i]))
vergleich = QaSimpleErgebnisVergleich(self.krit_id, klasse.klasse_id, klasse.bez, alt_aus, akt_aus)
neu_trend = QaSimpleErgebnisNeuTrend(self.krit_id, klasse.klasse_id, klasse.bez, alt_ges, vergleich, akt_aus, qa_klasse.gem_ges, qa_klasse.wab_ges)
if self.krit_id in KRIT_IDS_OF_KRITERIEN_WITH_KRITKLASSEN:
neu_trend.wab = self.np_klassen_neu_trend[i][0]
neu_trend.abg = self.np_klassen_neu_trend[i][1]
neu_trend.gig = self.np_klassen_neu_trend[i][2]
neu_trend.pstimmen = self.np_klassen_neu_trend[i][3:].tolist()
neu_trend.berechne_prozent()
vergleich = QaSimpleErgebnisVergleich(self.krit_id, klasse.klasse_id, klasse.bez, alt_ges, neu_trend)
self.qaklassen_alt_ges.append(alt_ges)
self.qaklassen_alt_aus.append(alt_aus)
self.qaklassen_akt_aus.append(akt_aus)
self.qaklassen_vergleich.append(vergleich)
self.qaklassen_neu_trend.append(neu_trend)
params = lambda np_klasse: (self.krit_id, 0, "", [None, self.gem_aus] + [int(j) for j in np_klasse.tolist()], self.gem_ges, self.wab_ges)
self.sum_alt_ges = SimpleQaErgebnis.from_erg_row(*params(self.np_sum_alt_ges))
self.sum_alt_aus = SimpleQaErgebnis.from_erg_row(*params(self.np_sum_alt_aus))
self.sum_akt_aus = SimpleQaErgebnis.from_erg_row(*params(self.np_sum_akt_aus))
self.sum_neu_trend = SimpleQaErgebnis.from_erg_row(*params(self.np_sum_neu_trend))
self.anz_parteien = len(self.hr.qalisten)
self.erg_akt_wab_ges = {}
for i, qa_klasse in enumerate(self.qa_klassen, start=1):
self.erg_akt_wab_ges[i] = (i, qa_klasse.gem_ges, qa_klasse.wab_ges)
self.erg_row_keys = ('krit_id', 'klasse_id', 'gem_ges', 'gem_aus', 'wab_ges', 'wab', 'abg', 'gig', 'p01', 'p02', 'p03', 'p04', 'p05', 'p06', 'p07', 'p08', 'p09','p10', 'p11', 'p12', 'p13', 'p14', 'p15', 'p16', 'p17', 'p18', 'p19', 'p20')
self.table_header = ['Krit', 'Klasse', 'GemGes', 'GemAus', 'WabGes', 'Wab', 'Abg', 'Gig', 'p01', 'p02', 'p03', 'p04', 'p05', 'p06', 'p07', 'p08', 'p09', 'p10', 'p11', 'p12', 'p13', 'p14', 'p15', 'p16', 'p17', 'p18', 'p19', 'p20']
self.klassen_bez = [klasse.bez for klasse in self.krit.klassen]
def before_pickle(self):
self.krit = None
self.wahl_akt = None
self.wahl_alt = None
def after_pickle(self):
self.krit = Kriterium.query.get(self.krit_id)
self.wahl_akt = BaseWahl.query.get(self.wahl_akt_id)
self.wahl_alt = BaseWahl.query.get(self.wahl_alt_id)
self.krit_bez = self.krit.bez
for npqaklasse in self.qa_klassen:
npqaklasse.after_pickle()
def as_dict(self):
keys_for_dict = ('wahl_akt_id', 'wahl_alt_id', 'krit_id', 'klasse_id', 'bl', 'is_fake', \
'np_klassen_alt_aus', 'np_klassen_akt_aus', 'np_klassen_alt_ges', 'np_klassen_akt_ges', 'np_klassen_diff', \
'np_alt_ges', 'np_alt_aus', 'np_akt_aus', 'np_akt_ges', 'np_diff')
d = {}
for key in keys_for_dict:
try:
item = getattr(self, key)
except AttributeError:
continue
if isinstance(item, (np.int64, np.int32)):
item = int(item)
if isinstance(item, np.ndarray):
item = item.tolist()
d[key] = item
d['qa_klassen'] = [k.as_dict() for k in self.qa_klassen]
return d
def asciitable(self, k):
if k.ndim == 1:
k = k.reshape((1, 24))
headers = ['', 'gkz', 'wab', 'abg', 'gig', 'p01', 'p02', 'p03', 'p04', 'p05', 'p06', 'p07', 'p08', 'p09', 'p10', 'p11', 'p12', 'p13', 'p14',
'p15', 'p16', 'p17', 'p18', 'p19', 'p20']
return(tabulate(k, headers=headers))
def get_npqakrit(krit_id, wahl_alt_id, wahl_akt_id, bl):
global redis_connection
key = get_redis_npqakrit_key(krit_id=krit_id, wahl_alt_id=wahl_alt_id, wahl_akt_id=wahl_akt_id, bl=bl)
if redis_connection is None:
initialize()
berechne_qa_for_hr(Hochrechnung.query.get(HR_ID))
logger.info("Load NpQaKrit: krit_id={krit_id}, wahl_akt_id={wahl_akt_id}, wahl_alt_id={wahl_alt_id}, bl={bl}".format(**locals()))
o = redis_connection.get(key)
if o is None:
npk = NpQaKrit(krit_id=krit_id, wahl_akt_id=wahl_akt_id, wahl_alt_id=wahl_alt_id, bl=bl)
add_key_to_redis_keys_for_qa(key)
npk.before_pickle()
redis_connection.set(key, pickle.dumps(npk))
else:
npk = pickle.loads(o)
npk.after_pickle()
return npk
def berechne_qa_for_hr():
redis_work_connection.flushdb()
initialize(force_reload_ergebnisse=True)
combinations_used = [tuple(map(int, k.decode('UTF-8').split(':')[1:])) for k in redis_connection.keys(pattern='npqakrit*')]
default_combinations = [tuple(map(int, k.split(':')[1:])) for k in DEFAULT_REDIS_NPQAKRIT_KEYS]
combinations = set(combinations_used + default_combinations)
for krit_id, wahl_alt_id, wahl_akt_id, bl in combinations:
logger.info("QA for krit_id={}, wahl_alt_id={}, wahl_akt_id={}, bl={}".format(krit_id, wahl_alt_id, wahl_akt_id, bl))
try:
npk = NpQaKrit(krit_id=krit_id, wahl_akt_id=wahl_akt_id, wahl_alt_id=wahl_alt_id, bl=bl)
key = get_redis_npqakrit_key(krit_id=krit_id, wahl_alt_id=wahl_alt_id, wahl_akt_id=wahl_akt_id, bl=bl)
npk.before_pickle()
redis_work_connection.set(key, pickle.dumps(npk))
except KriteriumWithoutKlassenException as e:
logger.warn(str(e))
def krit_is_equal(krit1 :QaKrit, krit2: NpQaKrit):
def h(msg):
print("{}: ".format(yellow(msg)), end="")
def ok():
print(green("OK"))
def failed():
print(red("Failed"))
def check(name):
import re
h(name)
if name.endswith(']'):
name, i, _ = re.split(r'\[|\]', name)
i = int(i)
_v1 = getattr(krit1, name)
_v2 = getattr(krit2, name)
v1 = _v1[i]
v2 = _v2[i]
else:
v1 = getattr(krit1, name)
v2 = getattr(krit2, name)
if v1 == v2:
print("{} ".format(v1), end='')
ok()
else:
failed()
print("krit1.{}: {}".format(name, v1))
print("krit2.{}: {}".format(name, v2))
check("has_ergebnisse")
check("krit_id")
check("anz_klassen")
for i in range(krit1.anz_klassen):
check("qaklassen_alt_ges[{}]".format(i))
check("qaklassen_alt_aus[{}]".format(i))
check("qaklassen_akt_aus[{}]".format(i))
check("qaklassen_neu_trend[{}]".format(i))
check("qaklassen_vergleich[{}]".format(i))
if __name__ == '__main__':
from wsgi.qa import Qa
initialize(force_reload_ergebnisse=True)
krit_id = 157
wahl_alt_id = 20
wahl_akt_id = 16
bl = 0
berechne_qa_for_hr()
n = NpQaKrit(krit_id=krit_id, wahl_alt_id=wahl_alt_id, wahl_akt_id=wahl_akt_id, bl=bl)
exit()
#n = get_npqakrit(krit_id=krit_id, wahl_alt_id=wahl_alt_id, wahl_akt_id=wahl_akt_id, bl=0)
m = NpQaKrit(krit_id=137, wahl_alt_id=wahl_alt_id, wahl_akt_id=wahl_akt_id, bl=0)
tirol = m.qa_klassen[6].np_neu_trend
print()
#qa, cached = Qa.get_or_create(krit_id=krit_id, wahl_alt_id=wahl_alt_id, wahl_akt_id=wahl_akt_id)
#krit_is_equal(n, qa)
#print(n.as_dict())
#json.dump(n.as_dict(), open('/opt/hr/wsgi/downloads/npqa.json', 'w'), indent=4, sort_keys=True)
#print(yaml.dump(n.as_dict(), indent=4, width=10000))
sys.exit()
krit_id = 2
klasse_id = 2
bl = 0
#npk.add_objects_to_session()
#db.session.commit()
#k = npk.qa_klassen[0]
#k.add_objects_to_session()
#sys.exit()
#from itertools import product
#kriterien = [krit.id for krit in hr.kriterien]
#wahlen_alt = [hr.wahl_alt.id] + [f.id for f in hr.fiktive_basen if f.id in (17, 19)]
#wahlen_akt = [hr.wahl_akt.id] + [f.id for f in hr.fiktive_basen if f.id in (17, 19)]
#combinations = product(kriterien, wahlen_alt, wahlen_akt)
#combinations = [combination for combination in combinations if combination[1] != combination[2]]
#print(len(combinations))
# from queue import Queue
# q = Queue()
# def worker():
# while True:
# item = q.get()
# if item is None:
# break
# krit, wahl_akt, wahl_alt = item
# sys.stdout.write('.')
# sys.stdout.flush()
# npk = NpQaKrit(krit, wahl_akt_id=wahl_akt.id, wahl_alt_id=wahl_alt.id, bl=bl)
# q.task_done()
#
# from threading import Thread
#
# threads = []
# num_worker_threads = 8
# for i in range(num_worker_threads):
# t = Thread(target=worker)
# t.start()
# threads.append(t)
#
# for item in combinations:
# q.put(item)
#
# q.join()
#
# for i in range(num_worker_threads):
# q.put(None)
# for t in threads:
# t.join()
for krit, wahl_akt, wahl_alt in combinations:
sys.stdout.write('.')
sys.stdout.flush()
get_npqakrit(hr, krit, wahl_alt, wahl_akt, bl)