temporalprofiles.py

# -*- coding: utf-8 -*-
"""
/***************************************************************************
                              EO Time Series Viewer
                              -------------------
        begin                : 2017-08-04
        git sha              : $Format:%H$
        copyright            : (C) 2017 by HU-Berlin
        email                : benjamin.jakimow@geo.hu-berlin.de
 ***************************************************************************/

/***************************************************************************
 *                                                                         *
 *   This program is free software; you can redistribute it and/or modify  *
 *   it under the terms of the GNU General Public License as published by  *
 *   the Free Software Foundation; either version 2 of the License, or     *
 *   (at your option) any later version.                                   *
 *                                                                         *
 ***************************************************************************/
"""
# noinspection PyPep8Naming

import os, sys, pickle, datetime, re, collections

from collections import OrderedDict
from qgis.gui import *
from qgis.core import *
from qgis.PyQt.QtCore import *
from qgis.PyQt.QtGui import *
from qgis.PyQt.QtWidgets import *
import numpy as np
from osgeo import ogr, osr, gdal
from .externals import pyqtgraph as pg
from .externals.pyqtgraph import functions as fn, AxisItem, ScatterPlotItem, SpotItem, GraphicsScene
from .externals.qps.plotstyling.plotstyling import PlotStyle

from .timeseries import TimeSeries, TimeSeriesDate, SensorInstrument, TimeSeriesSource
from .pixelloader import PixelLoader, PixelLoaderTask
from .utils import *
from .externals.qps.speclib.spectrallibraries import createQgsField


LABEL_EXPRESSION_2D = 'DN or Index'
LABEL_TIME = 'Date'
DEBUG = False
OPENGL_AVAILABLE = False
DEFAULT_SAVE_PATH = None
DEFAULT_CRS = QgsCoordinateReferenceSystem('EPSG:4326')

FN_ID = 'fid'
FN_X = 'x'
FN_Y = 'y'
FN_NAME = 'name'

FN_DOY = 'DOY'
FN_DTG = 'DTG'
FN_IS_NODATA ='is_nodata'
FN_GEO_X = 'geo_x'
FN_GEO_Y = 'geo_y'
FN_PX_X = 'px_x'
FN_PX_Y = 'px_y'

#FN_N_TOTAL = 'n'
#FN_N_NODATA = 'no_data'
#FN_N_LOADED = 'loaded'
#FN_N_LOADED_PERCENT = 'percent'

regBandKey = re.compile(r"(?<!\w)b\d+(?!\w)", re.IGNORECASE)
regBandKeyExact = re.compile(r'^' + regBandKey.pattern + '$', re.IGNORECASE)



try:
    import OpenGL

    OPENGL_AVAILABLE = True

except:
    pass



def temporalProfileFeatureFields(sensor: SensorInstrument, singleBandOnly=False) -> QgsFields:
    """
    Returns the fields of a single temporal profile
    :return:
    """
    assert isinstance(sensor, SensorInstrument)
    fields = QgsFields()
    fields.append(createQgsField(FN_DTG, '2011-09-12', comment='Date-time-group'))
    fields.append(createQgsField(FN_DOY, 42, comment='Day-of-year'))
    fields.append(createQgsField(FN_GEO_X, 12.1233, comment='geo-coordinate x/east value'))
    fields.append(createQgsField(FN_GEO_Y, 12.1233, comment='geo-coordinate y/north value'))
    fields.append(createQgsField(FN_PX_X, 42, comment='pixel-coordinate x index'))
    fields.append(createQgsField(FN_PX_Y, 24, comment='pixel-coordinate y index'))


    for b in range(sensor.nb):
        bandKey = bandIndex2bandKey(b)
        fields.append(createQgsField(bandKey, 1.0, comment='value band {}'.format(b+1)))

    return fields

def sensorExampleQgsFeature(sensor:SensorInstrument, singleBandOnly=False)->QgsFeature:
    """
    Returns an exemplary QgsFeature with value for a specific sensor
    :param sensor: SensorInstrument
    :param singleBandOnly:
    :return:
    """
    # populate with exemplary band values (generally stored as floats)

    fields = temporalProfileFeatureFields(sensor)
    f = QgsFeature(fields)
    pt = QgsPointXY(12.34567, 12.34567)
    f.setGeometry(QgsGeometry.fromPointXY(pt))
    f.setAttribute(FN_GEO_X, pt.x())
    f.setAttribute(FN_GEO_Y, pt.y())
    f.setAttribute(FN_PX_X, 1)
    f.setAttribute(FN_PX_Y, 1)
    dtg = datetime.date.today()
    doy = dateDOY(dtg)
    f.setAttribute(FN_DTG, str(dtg))
    f.setAttribute(FN_DOY, doy)

    for b in range(sensor.nb):
        bandKey = bandIndex2bandKey(b)
        f.setAttribute(bandKey, 1.0)

    return f




def dateDOY(date):
    if isinstance(date, np.datetime64):
        date = date.astype(datetime.date)
    return date.timetuple().tm_yday

def daysPerYear(year):

    if isinstance(year, np.datetime64):
        year = year.astype(datetime.date)
    if isinstance(year, datetime.date):
        year = year.timetuple().tm_year

    return dateDOY(datetime.date(year=year, month=12, day=31))

def date2num(d):
    #kindly taken from https://stackoverflow.com/questions/6451655/python-how-to-convert-datetime-dates-to-decimal-years
    if isinstance(d, np.datetime64):
        d = d.astype(datetime.datetime)

    if isinstance(d, QDate):
        d = datetime.date(d.year(), d.month(), d.day())

    assert isinstance(d, datetime.date)

    yearDuration = daysPerYear(d)
    yearElapsed = d.timetuple().tm_yday
    fraction = float(yearElapsed) / float(yearDuration)
    if fraction == 1.0:
        fraction = 0.9999999
    return float(d.year) + fraction

def num2date(n, dt64=True, qDate=False):
    n = float(n)
    if n < 1:
        n += 1

    year = int(n)
    fraction = n - year
    yearDuration = daysPerYear(year)
    yearElapsed = fraction * yearDuration

    import math
    doy = round(yearElapsed)
    if doy < 1:
        doy = 1
    try:
        date = datetime.date(year, 1, 1) + datetime.timedelta(days=doy-1)
    except:
        s = ""
    if qDate:
        return QDate(date.year, date.month, date.day)
    if dt64:
        return np.datetime64(date)
    else:
        return date



    #return np.datetime64('{:04}-01-01'.format(year), 'D') + np.timedelta64(int(yearElapsed), 'D')





def bandIndex2bandKey(i : int):
    assert i >= 0
    return 'b{}'.format(i + 1)

def bandKey2bandIndex(key: str):
    match = regBandKeyExact.search(key)
    assert match
    idx = int(match.group()[1:]) - 1
    return idx



class DateTimePlotWidget(pg.PlotWidget):
    """
    Subclass of PlotWidget
    """

    def __init__(self, parent=None):
        """
        Constructor of the widget
        """
        super(DateTimePlotWidget, self).__init__(parent)
        self.plotItem = pg.PlotItem(
            axisItems={'bottom':DateTimeAxis(orientation='bottom')}
            ,viewBox=DateTimeViewBox()
        )
        self.setCentralItem(self.plotItem)
        #self.xAxisInitialized = False

        pi = self.getPlotItem()
        pi.getAxis('bottom').setLabel(LABEL_TIME)
        pi.getAxis('left').setLabel(LABEL_EXPRESSION_2D)

        self.mInfoColor = QColor('yellow')
        self.mCrosshairLineV = pg.InfiniteLine(angle=90, movable=False)
        self.mCrosshairLineH = pg.InfiniteLine(angle=0, movable=False)
        self.mInfoLabelCursor = pg.TextItem(text='<cursor position>', anchor=(1.0, 0.0))
        self.mInfoLabelCursor.setColor(QColor('yellow'))

        self.scene().addItem(self.mInfoLabelCursor)
        self.mInfoLabelCursor.setParentItem(self.getPlotItem())
        #self.plot2DLabel.setAnchor()
        #self.plot2DLabel.anchor(itemPos=(0, 0), parentPos=(0, 0), offset=(0, 0))
        pi.addItem(self.mCrosshairLineV, ignoreBounds=True)
        pi.addItem(self.mCrosshairLineH, ignoreBounds=True)

        assert isinstance(self.scene(), pg.GraphicsScene)
        self.proxy2D = pg.SignalProxy(self.scene().sigMouseMoved, rateLimit=60, slot=self.onMouseMoved2D)

    def resetViewBox(self):
        self.plotItem.getViewBox().autoRange()



    def onMouseMoved2D(self, evt):
        pos = evt[0]  ## using signal proxy turns original arguments into a tuple

        plotItem = self.getPlotItem()
        if plotItem.sceneBoundingRect().contains(pos):
            vb = plotItem.vb
            assert isinstance(vb, DateTimeViewBox)
            mousePoint = vb.mapSceneToView(pos)
            x = mousePoint.x()
            if x >= 0:
                y = mousePoint.y()
                date = num2date(x)
                doy = dateDOY(date)
                plotItem.vb.updateCurrentDate(num2date(x, dt64=True))
                self.mInfoLabelCursor.setText('DN {:0.2f}\nDate {}\nDOY {}'.format(
                                              mousePoint.y(), date, doy),
                                              color=self.mInfoColor)

                s = self.size()
                pos = QPointF(s.width(), 0)
                self.mInfoLabelCursor.setVisible(vb.mActionShowCursorValues.isChecked())
                self.mInfoLabelCursor.setPos(pos)

                b = vb.mActionShowCrosshair.isChecked()
                self.mCrosshairLineH.setVisible(b)
                self.mCrosshairLineV.setVisible(b)
                self.mCrosshairLineH.pen.setColor(self.mInfoColor)
                self.mCrosshairLineV.pen.setColor(self.mInfoColor)
                self.mCrosshairLineV.setPos(mousePoint.x())
                self.mCrosshairLineH.setPos(mousePoint.y())


class DateTimeAxis(pg.AxisItem):

    def __init__(self, *args, **kwds):
        super(DateTimeAxis, self).__init__(*args, **kwds)
        self.setRange(1,3000)
        self.enableAutoSIPrefix(False)
        self.labelAngle = 0

    def logTickStrings(self, values, scale, spacing):
        s = ""


    def tickStrings(self, values, scale, spacing):
        strns = []

        if len(values) == 0:
            return []
        #assert isinstance(values[0],

        values = [num2date(v) if v > 0 else num2date(1) for v in values]
        rng = max(values)-min(values)
        ndays = rng.astype(int)

        strns = []

        for v in values:
            if ndays == 0:
                strns.append(v.astype(str))
            else:
                strns.append(v.astype(str))

        return strns

    def tickValues(self, minVal, maxVal, size):
        d = super(DateTimeAxis, self).tickValues(minVal, maxVal, size)

        return d


    def drawPicture(self, p, axisSpec, tickSpecs, textSpecs):


        p.setRenderHint(p.Antialiasing, False)
        p.setRenderHint(p.TextAntialiasing, True)

        ## draw long line along axis
        pen, p1, p2 = axisSpec
        p.setPen(pen)
        p.drawLine(p1, p2)
        p.translate(0.5, 0)  ## resolves some damn pixel ambiguity

        ## draw ticks
        for pen, p1, p2 in tickSpecs:
            p.setPen(pen)
            p.drawLine(p1, p2)


        ## Draw all text
        if self.tickFont is not None:
            p.setFont(self.tickFont)
        p.setPen(self.pen())

        #for rect, flags, text in textSpecs:
        #    p.drawText(rect, flags, text)
        #    # p.drawRect(rect)

        #see https://github.com/pyqtgraph/pyqtgraph/issues/322
        for rect, flags, text in textSpecs:
            p.save()  # save the painter state
            p.translate(rect.center())   # move coordinate system to center of text rect
            p.rotate(self.labelAngle)  # rotate text
            p.translate(-rect.center())  # revert coordinate system
            p.drawText(rect, flags, text)
            p.restore()  # restore the painter state



class DateTimeViewBox(pg.ViewBox):
    """
    Subclass of ViewBox
    """
    sigMoveToDate = pyqtSignal(np.datetime64)
    sigMoveToLocation = pyqtSignal(SpatialPoint)
    def __init__(self, parent=None):
        """
        Constructor of the CustomViewBox
        """
        super(DateTimeViewBox, self).__init__(parent)
        #self.menu = None # Override pyqtgraph ViewBoxMenu
        #self.menu = self.getMenu() # Create the menu
        #self.menu = None
        self.mCurrentDate = np.datetime64('today')

        self.mXAxisUnit = 'date'
        xAction = [a for a in self.menu.actions() if a.text() == 'X Axis'][0]
        yAction = [a for a in self.menu.actions() if a.text() == 'Y Axis'][0]

        menuXAxis = self.menu.addMenu('X Axis')
        #define the widget to set X-Axis options
        frame = QFrame()
        l = QGridLayout()

        frame.setLayout(l)
        #l.addWidget(self, QWidget, int, int, alignment: Qt.Alignment = 0): not enough arguments
        self.rbXManualRange = QRadioButton('Manual')
        self.dateEditX0 = QDateEdit()
        self.dateEditX0.setDisplayFormat('yyyy-MM-dd')
        self.dateEditX0.setToolTip('Start time')
        self.dateEditX0.setCalendarPopup(True)
        self.dateEditX0.dateChanged.connect(self.updateXRange)
        self.dateEditX1 = QDateEdit()
        self.dateEditX1.setDisplayFormat('yyyy-MM-dd')
        self.dateEditX0.setToolTip('End time')
        self.dateEditX1.setCalendarPopup(True)
        self.dateEditX1.dateChanged.connect(self.updateXRange)

        self.rbXAutoRange = QRadioButton('Auto')
        self.rbXAutoRange.setChecked(True)
        self.rbXAutoRange.toggled.connect(self.updateXRange)


        l.addWidget(self.rbXManualRange, 0,0)
        l.addWidget(self.dateEditX0, 0,1)
        l.addWidget(self.dateEditX1, 0,2)
        l.addWidget(self.rbXAutoRange, 1, 0)

        l.setMargin(1)
        l.setSpacing(1)
        frame.setMinimumSize(l.sizeHint())
        wa = QWidgetAction(menuXAxis)
        wa.setDefaultWidget(frame)
        menuXAxis.addAction(wa)


        self.menu.insertMenu(xAction, menuXAxis)
        self.menu.removeAction(xAction)

        self.mActionMoveToDate = self.menu.addAction('Move to {}'.format(self.mCurrentDate))
        self.mActionMoveToDate.triggered.connect(lambda *args : self.sigMoveToDate.emit(self.mCurrentDate))

        #self.mActionMoveToProfile = self.menu.addAction('Move to profile location')
        #self.mActionMoveToProfile.triggered.connect(lambda *args: self.sigM.emit(self.mCurrentDate))

        self.mActionShowCrosshair = self.menu.addAction('Show Crosshair')
        self.mActionShowCrosshair.setCheckable(True)
        self.mActionShowCrosshair.setChecked(True)
        self.mActionShowCursorValues = self.menu.addAction('Show Mouse values')
        self.mActionShowCursorValues.setCheckable(True)
        self.mActionShowCursorValues.setChecked(True)

    sigXAxisUnitChanged = pyqtSignal(str)
    def setXAxisUnit(self, unit):
        assert unit in ['date', 'doy']
        old = self.mXAxisUnit
        self.mXAxisUnit = unit
        if old != self.mXAxisUnit:
            self.sigXAxisUnitChanged.emit(self.mXAxisUnit)

    def xAxisUnit(self):
        return self.mXAxisUnit

    def updateXRange(self, *args):
        isAutoRange = self.rbXAutoRange.isChecked()
        self.enableAutoRange('x', isAutoRange)

        self.dateEditX0.setEnabled(not isAutoRange)
        self.dateEditX1.setEnabled(not isAutoRange)

        if not isAutoRange:
            t0 = date2num(self.dateEditX0.date())
            t1 = date2num(self.dateEditX1.date())
            t0 = min(t0, t1)
            t1 = max(t0, t1)

            self.setXRange(t0, t1)

    def updateCurrentDate(self, date):
        if isinstance(date, np.datetime64):
            self.mCurrentDate = date
            self.mActionMoveToDate.setData(date)
            self.mActionMoveToDate.setText('Move maps to {}'.format(date))



    def raiseContextMenu(self, ev):

        pt = self.mapDeviceToView(ev.pos())
        self.updateCurrentDate(num2date(pt.x(), dt64=True))

        plotDataItems = [item for item in self.scene().itemsNearEvent(ev) if isinstance(item, ScatterPlotItem) and isinstance(item.parentItem(), TemporalProfilePlotDataItem)]


        xRange, yRange = self.viewRange()
        t0 = num2date(xRange[0], qDate=True)
        t1 = num2date(xRange[1], qDate=True)
        self.dateEditX0.setDate(t0)
        self.dateEditX1.setDate(t1)

        menu = self.getMenu(ev)

        if len(plotDataItems) > 0:
            s = ""

        self.scene().addParentContextMenus(self, menu, ev)
        menu.exec_(ev.screenPos().toPoint())



class TemporalProfile(QObject):


    sigNameChanged = pyqtSignal(str)
    #sigDataChanged = pyqtSignal()

    def __init__(self, layer, fid:int, geometry:QgsGeometry):
        super(TemporalProfile, self).__init__()
        assert isinstance(geometry, QgsGeometry)
        assert isinstance(layer, TemporalProfileLayer)
        assert fid >= 0


        self.mID = fid
        self.mLayer = layer
        self.mTimeSeries = layer.timeSeries()
        assert isinstance(self.mTimeSeries, TimeSeries)
        self.mData = {}
        self.mUpdated = False
        self.mLoaded = self.mLoadedMax = self.mNoData = 0

        for tsd in self.mTimeSeries:
            assert isinstance(tsd, TimeSeriesDate)
            meta = {FN_DOY: tsd.mDOY,
                    FN_DTG: str(tsd.mDate),
                    FN_IS_NODATA: False}

            self.updateData(tsd, meta, skipStatusUpdate=True)

        self.mGEOM_CACHE = dict()

    def printData(self, sensor:SensorInstrument=None):
        """
        Prints the entire temporal profile. For debug purposes.
        """
        for tsd in sorted(self.mData.keys()):
            assert isinstance(tsd, TimeSeriesDate)
            data = self.mData[tsd]
            if isinstance(sensor, SensorInstrument) and tsd.sensor() != sensor:
                continue
            assert isinstance(data, dict)
            info = '{}:{}={}'.format(tsd.date(), tsd.sensor().name(), str(data))
            print(info)

    def __hash__(self):
        return hash('{}{}'.format(self.mID, self.mLayer.layerId()))

    def __eq__(self, other):
        """
        Two temporal profiles are equal if they have the same feature id and source layer
        :param other:
        :return:
        """

        if not isinstance(other, TemporalProfile):
            return False

        return other.mID == self.mID and self.mLayer == other.mLayer

    def geometry(self, crs:QgsCoordinateReferenceSystem=None)->QgsGeometry:
        """
        Returns the geometry
        :param crs:
        :return: QgsGeometry. usually a  QgsPoint
        """
        g = self.mLayer.getFeature(self.mID).geometry()

        if not isinstance(g, QgsGeometry):
            return None

        if isinstance(crs, QgsCoordinateReferenceSystem) and crs != self.mLayer.crs():
            trans = QgsCoordinateTransform()
            trans.setSourceCrs(self.mLayer.crs())
            trans.setDestinationCrs(crs)
            g.transform(trans)
        return g


    def coordinate(self)->SpatialPoint:
        """
        Returns the profile coordinate
        :return:
        """
        x, y = self.geometry().asPoint()
        return SpatialPoint(self.mLayer.crs(), x, y)

    def id(self):
        """Feature ID in connected QgsVectorLayer"""
        return self.mID

    def attribute(self, key:str):
        f = self.mLayer.getFeature(self.mID)
        return f.attribute(f.fieldNameIndex(key))

    def setAttribute(self, key:str, value):
        f = self.mLayer.getFeature(self.id())

        b = self.mLayer.isEditable()
        self.mLayer.startEditing()
        self.mLayer.changeAttributeValue(f.id(), f.fieldNameIndex(key), value)
        self.mLayer.saveEdits(leaveEditable=b)

    def name(self):
        return self.attribute('name')

    def setName(self, name:str):
        self.setAttribute('name', name)

    def data(self):
        return self.mData

    def timeSeries(self):
        return self.mTimeSeries


    def loadMissingData(self):
        """
        Loads the missing data for this profile (synchronous eexecution, may take some time).
        """
        tasks = []
        for tsd in self.mTimeSeries:
            assert isinstance(tsd, TimeSeriesDate)
            missingIndices = self.missingBandIndices(tsd)

            if len(missingIndices) > 0:
                for tss in tsd:
                    assert isinstance(tss, TimeSeriesSource)
                    if tss.spatialExtent().contains(self.coordinate().toCrs(tss.crs())):
                        task = TemporalProfileLoaderTask(tss, [self], bandIndices=missingIndices)
                        tasks.append(task)

        results = doLoadTemporalProfileTasks(TaskMock(), pickle.dumps(tasks))
        ts = self.timeSeries()
        for result in pickle.loads(results):
            assert isinstance(result, TemporalProfileLoaderTask)
            tsd = ts.getTSD(result.mSourcePath)
            assert isinstance(tsd, TimeSeriesDate)
            for tpId, data in result.mRESULTS.items():
                if tpId == self.id():
                    self.updateData(tsd, data)

    def missingBandIndices(self, tsd, requiredIndices=None):
        """
        Returns the band indices [0, sensor.nb) that have not been loaded yet.
        :param tsd: TimeSeriesDate of interest
        :param requiredIndices: optional subset of possible band-indices to return the missing ones from.
        :return: [list-of-indices]
        """
        assert isinstance(tsd, TimeSeriesDate)
        if requiredIndices is None:
            requiredIndices = list(range(tsd.mSensor.nb))
        requiredIndices = [i for i in requiredIndices if i >= 0 and i < tsd.mSensor.nb]

        existingBandIndices = [bandKey2bandIndex(k) for k in self.data(tsd).keys() if regBandKeyExact.search(k)]

        if FN_PX_X not in self.data(tsd).keys() and len(requiredIndices) == 0:
            requiredIndices.append(0)

        return [i for i in requiredIndices if i not in existingBandIndices]


    def plot(self):


        for sensor in self.mTimeSeries.sensors():
            assert isinstance(sensor, SensorInstrument)

            plotStyle = TemporalProfile2DPlotStyle(self)
            plotStyle.setSensor(sensor)

            pi = TemporalProfilePlotDataItem(plotStyle)
            pi.setClickable(True)
            pw = pg.plot(title=self.name())
            pw.plotItem().addItem(pi)
            pi.setColor('green')
            pg.QAPP.exec_()

    def updateData(self, tsd, values, skipStatusUpdate=False):
        assert isinstance(tsd, TimeSeriesDate)
        assert isinstance(values, dict)

        if tsd not in self.mData.keys():
            self.mData[tsd] = dict()

        values2 = self.mData.get(tsd)
        assert isinstance(values2, dict)
        for k, v in values.items():
            if v in [None, np.NaN] and k not in values2.keys():
                    values2[k] = v
            else:
                values2[k] = v

        #self.mData[tsd].update(values)

        if not skipStatusUpdate:
            #self.updateLoadingStatus()
            self.mUpdated = True
            #self.sigDataChanged.emit()

    def resetUpdatedFlag(self):
        self.mUpdated = False

    def updated(self):
        return self.mUpdated

    def dataFromExpression(self, sensor:SensorInstrument, expression:str, dateType='date'):
        assert dateType in ['date', 'doy']
        x = []
        y = []


        if not isinstance(expression, QgsExpression):
            expression = QgsExpression(expression)
        assert isinstance(expression, QgsExpression)
        expression = QgsExpression(expression)



        sensorTSDs = sorted([tsd for tsd in self.mData.keys() if tsd.sensor() == sensor])

        # define required QgsFields
        fields = temporalProfileFeatureFields(sensor)

        geo_x = self.geometry().centroid().get().x()
        geo_y = self.geometry().centroid().get().y()

        for i, tsd in enumerate(sensorTSDs):
            assert isinstance(tsd, TimeSeriesDate)
            data = self.mData[tsd]

            if dateType == 'date':
                xValue = date2num(tsd.mDate)
            elif dateType == 'doy':
                xValue = tsd.mDOY

            context = QgsExpressionContext()
            context.setFields(fields)

            f = QgsFeature(fields)

            # set static properties (same for all TSDs)
            f.setGeometry(QgsGeometry(self.geometry()))
            f.setAttribute(FN_GEO_X, geo_x)
            f.setAttribute(FN_GEO_Y, geo_y)

            # set TSD specific properties
            f.setAttribute(FN_DOY, tsd.doy())
            f.setAttribute(FN_DTG, str(tsd.date()))

            for fn in fields.names():
                if fn in data.keys():
                    setQgsFieldValue(f, fn, data[fn])

            context.setFeature(f)

            yValue = expression.evaluate(context)

            if yValue in [None, QVariant()]:
                yValue = np.NaN

            y.append(yValue)
            x.append(xValue)

        #return np.asarray(x), np.asarray(y)
        assert len(x) == len(y)
        return x, y

    def data(self, tsd):
        assert isinstance(tsd, TimeSeriesDate)
        if self.hasData(tsd):
            return self.mData[tsd]
        else:
            return {}


    def loadingStatus(self):
        """
        Returns the loading status in terms of single pixel values.
        nLoaded = sum of single band values
        nLoadedMax = potential maximum of band values that might be loaded
        :return: (nLoaded, nLoadedMax)
        """
        return self.mLoaded, self.mNoData, self.mLoadedMax

    #def updateLoadingStatus(self):
    #    """
    #    Calculates the loading status in terms of single pixel values.
    #    nMax is the sum of all bands over each TimeSeriesDate and Sensors
    #    """
    """
        self.mLoaded = 0
        self.mLoadedMax = 0
        self.mNoData = 0

        for tsd in self.mTimeSeries:
            assert isinstance(tsd, TimeSeriesDate)
            nb = tsd.mSensor.nb

            self.mLoadedMax += nb
            if self.hasData(tsd):
                if self.isNoData(tsd):
                    self.mNoData += nb
                else:
                    self.mLoaded += len([k for k in self.mData[tsd].keys() if regBandKey.search(k)])

        f = self.mLayer.getFeature(self.id())

        b = self.mLayer.isEditable()
        self.mLayer.startEditing()
        # self.mLayer.changeAttributeValue(f.id(), f.fieldNameIndex(FN_N_NODATA), self.mNoData)
        # self.mLayer.changeAttributeValue(f.id(), f.fieldNameIndex(FN_N_TOTAL), self.mLoadedMax)
        # self.mLayer.changeAttributeValue(f.id(), f.fieldNameIndex(FN_N_LOADED), self.mLoaded)
        # if self.mLoadedMax > 0:
        #     self.mLayer.changeAttributeValue(f.id(), f.fieldNameIndex(FN_N_LOADED_PERCENT), round(100. * float(self.mLoaded + self.mNoData) / self.mLoadedMax, 2))

        self.mLayer.saveEdits(leaveEditable=b)
        s = ""
    """

    def isNoData(self, tsd):
        assert isinstance(tsd, TimeSeriesDate)
        return self.mData[tsd][FN_IS_NODATA]

    def hasData(self, tsd):
        assert isinstance(tsd, TimeSeriesDate)
        return tsd in self.mData.keys()

    def __repr__(self):
        return 'TemporalProfile {} "{}"'.format(self.id(), self.name())


class TemporalProfilePlotStyleBase(PlotStyle):

    sigStyleUpdated = pyqtSignal()
    sigDataUpdated = pyqtSignal()
    sigExpressionUpdated = pyqtSignal()
    sigSensorChanged = pyqtSignal(SensorInstrument)

    def __init__(self, parent=None, temporalProfile=None):
        super(TemporalProfilePlotStyleBase, self).__init__()
        self.mSensor = None
        self.mTP = None
        self.mExpression = 'b1'
        self.mPlotItems = []
        self.mIsVisible = True
        self.mShowLastLocation = True

        if isinstance(temporalProfile, TemporalProfile):
            self.setTemporalProfile(temporalProfile)

    def showLastLocation(self)->bool:
        """
        """
        return self.mShowLastLocation

    def isPlotable(self):
        return self.isVisible() and isinstance(self.temporalProfile(), TemporalProfile) and isinstance(self.sensor(), SensorInstrument)

    def createPlotItem(self):
        raise NotImplementedError()

    def temporalProfile(self):
        return self.mTP

    def setTemporalProfile(self, temporalPofile):

        b = temporalPofile != self.mTP
        self.mTP = temporalPofile
        if temporalPofile in [None, QVariant()]:
            s  =""
        else:
            assert isinstance(temporalPofile, TemporalProfile)
        if b:
            self.updateDataProperties()

    def setSensor(self, sensor):
        assert sensor is None or isinstance(sensor, SensorInstrument)
        b = sensor != self.mSensor
        self.mSensor = sensor
        if b:
            self.update()
            self.sigSensorChanged.emit(sensor)

    def sensor(self):
        return self.mSensor

    def updateStyleProperties(self):
        raise NotImplementedError()

    def updateDataProperties(self):
        raise NotImplementedError()

    def update(self):
        self.updateDataProperties()

    def setExpression(self, exp):
        assert isinstance(exp, str)
        b = self.mExpression != exp
        self.mExpression = exp
        self.updateDataProperties()
        if b:
            self
            self.sigExpressionUpdated.emit()

    def expression(self):
        return self.mExpression

    def __reduce_ex__(self, protocol):
        return self.__class__, (), self.__getstate__()

    def __getstate__(self):
        result = super(TemporalProfile2DPlotStyle, self).__getstate__()
        #remove
        del result['mTP']
        del result['mSensor']

        return result

    def isVisible(self):
        return self.mIsVisible

    def setVisibility(self, b):
        assert isinstance(b, bool)
        old = self.isVisible()
        self.mIsVisible = b

        if b != old:
            self.updateStyleProperties()
            #self.update()

    def copyFrom(self, plotStyle):
        if isinstance(plotStyle, PlotStyle):
            super(TemporalProfilePlotStyleBase, self).copyFrom(plotStyle)
            self.updateStyleProperties()

        if isinstance(plotStyle, TemporalProfilePlotStyleBase):
            self.setExpression(plotStyle.expression())
            self.setSensor(plotStyle.sensor())
            self.setTemporalProfile(plotStyle.temporalProfile())
            self.updateDataProperties()




class TemporalProfile2DPlotStyle(TemporalProfilePlotStyleBase):


    def __init__(self, temporalProfile=None):
        super(TemporalProfile2DPlotStyle, self).__init__(temporalProfile=temporalProfile)
        #PlotStyle.__init__(self)
       #TemporalProfilePlotStyleBase.__init__(self, temporalProfile=temporalProfile)

    def createPlotItem(self, plotWidget):
        pdi = TemporalProfilePlotDataItem(self)
        self.mPlotItems.append(pdi)
        return pdi

    def updateStyleProperties(self):
        for pdi in self.mPlotItems:
            assert isinstance(pdi, TemporalProfilePlotDataItem)
            pdi.updateStyle()

    def updateDataProperties(self):
        for pdi in self.mPlotItems:
            assert isinstance(pdi, TemporalProfilePlotDataItem)
            pdi.updateDataAndStyle()









class TemporalProfileLoaderTask(object):
    """
    An object to loading temporal profile values from a *single* raster source.
    """

    def __init__(self, tss:TimeSeriesSource, temporalProfiles: list, bandIndices=None):

        assert isinstance(temporalProfiles, list)

        self.mId = ''
        self.mTSS = tss
        # assert isinstance(source, str) or isinstance(source, unicode)
        self.mSourcePath = tss.uri()
        self.mGeometries = {}
        self.mRESULTS = {}
        self.mERRORS = []

        for tp in temporalProfiles:
            assert isinstance(tp, TemporalProfile)
            # save geometry as WKT to be pickable
            self.mGeometries[tp.id()] = tp.geometry(tss.crs()).asWkt()

        if bandIndices is None:
            bandIndices = list(range(tss.nb))
        self.mBandIndices = bandIndices




def doLoadTemporalProfileTasks(qgsTask:QgsTask, dump):

    assert isinstance(qgsTask, QgsTask)
    tasks = pickle.loads(dump)