From 0519e87599e5f4c20dc24a5d576b823bb83e4e56 Mon Sep 17 00:00:00 2001
From: "benjamin.jakimow@geo.hu-berlin.de" <q8DTkxUg-BB>
Date: Mon, 12 Mar 2018 13:42:10 +0100
Subject: [PATCH] fixel pixeloader error
---
test/test_pixelloader.py | 29 ++++--
test/test_spectrallibraries.py | 41 +++++++-
timeseriesviewer/pixelloader.py | 117 ++++++++++++++---------
timeseriesviewer/profilevisualization.py | 6 +-
timeseriesviewer/temporalprofiles.py | 31 +++---
5 files changed, 149 insertions(+), 75 deletions(-)
diff --git a/test/test_pixelloader.py b/test/test_pixelloader.py
index d21febb8..c1ec4e6a 100644
--- a/test/test_pixelloader.py
+++ b/test/test_pixelloader.py
@@ -22,6 +22,10 @@ from timeseriesviewer import file_search
QGIS_APP = initQgisApplication()
+def onDummy(*args):
+ print(('dummy', args))
+
+
class PixelLoaderTest(unittest.TestCase):
"""Test translations work."""
@@ -141,12 +145,14 @@ class PixelLoaderTest(unittest.TestCase):
ext = SpatialExtent.fromRasterSource(files[0])
x, y = ext.center()
- geoms = [
+ geoms1 = [
# SpatialPoint(ext.crs(), 681151.214,-752388.476), #nodata in Img_2014_04_29_LE72270652014119CUB00_BOA
SpatialExtent(ext.crs(), x + 10000, y, x + 12000, y + 70), # out of image
SpatialExtent(ext.crs(), x, y, x + 10000, y + 70),
- SpatialPoint(ext.crs(), x, y),
- SpatialPoint(ext.crs(), x + 250, y + 70)]
+ ]
+
+ geoms2 = [SpatialPoint(ext.crs(), x, y),
+ SpatialPoint(ext.crs(), x + 250, y + 70)]
from multiprocessing import Pool
@@ -157,8 +163,6 @@ class PixelLoaderTest(unittest.TestCase):
PL = PixelLoader()
- def onDummy(*args):
- print(('dummy', args))
def onTimer(*args):
print(('TIMER', PL))
@@ -170,12 +174,21 @@ class PixelLoaderTest(unittest.TestCase):
PL.sigLoadingStarted.connect(lambda: onDummy('started'))
PL.sigPixelLoaded.connect(lambda: onDummy('px loaded'))
- tasks = []
+ tasks1 = []
+ for i, f in enumerate(files):
+ kwargs = {'myid': 'myID{}'.format(i)}
+ tasks1.append(PixelLoaderTask(f, geoms1, bandIndices=None, **kwargs))
+
+ tasks2 = []
for i, f in enumerate(files):
kwargs = {'myid': 'myID{}'.format(i)}
- tasks.append(PixelLoaderTask(f, geoms, bandIndices=None, **kwargs))
+ tasks2.append(PixelLoaderTask(f, geoms2, bandIndices=None, **kwargs))
+
+ PL.startLoading(tasks1)
+ PL.startLoading(tasks2)
+
+ print('DONE')
- PL.startLoading(tasks)
diff --git a/test/test_spectrallibraries.py b/test/test_spectrallibraries.py
index 5bfe2199..e4527651 100644
--- a/test/test_spectrallibraries.py
+++ b/test/test_spectrallibraries.py
@@ -17,7 +17,7 @@
***************************************************************************
"""
# noinspection PyPep8Naming
-import os, sys, unittest
+import os, sys, unittest, tempfile
from timeseriesviewer.utils import initQgisApplication
qapp = initQgisApplication()
@@ -45,11 +45,44 @@ class TestInit(unittest.TestCase):
self.SP = spec1
def test_spectralLibrary(self):
+
+ spec1 = SpectralProfile()
+ spec1.setValues([0, 4, 3, 2, 1], ['-'], [450, 500, 750, 1000, 1500], 'nm')
+
+ spec2 = SpectralProfile()
+ spec2.setValues([3, 2, 1, 0, 1], ['-'], [450, 500, 750, 1000, 1500], 'nm')
+
sl = SpectralLibrary()
- sl.addProfile(self.SP)
+ sl.addProfiles([spec1, spec2])
+ self.assertEqual(len(sl),2)
+ self.assertEqual(sl[0], spec1)
+
+
+ tempDir = tempfile.gettempdir()
+ pathESL = tempfile.mktemp(prefix='speclib.', suffix='.esl')
+ pathCSV = tempfile.mktemp(prefix='speclib.', suffix='.csv')
+ try:
+ sl.exportProfiles(pathESL)
+ except Exception as ex:
+ self.fail('Unable to write ESL. {}'.format(ex))
+
+ try:
+ sl2 = SpectralLibrary.readFrom(pathESL)
+ except Exception as ex:
+ self.fail('Unable to read ESL. {}'.format(ex))
+
+
+ try:
+ sl.exportProfiles(pathCSV)
+ except Exception as ex:
+ self.fail('Unable to write CSV. {}'.format(ex))
+
+ try:
+ sl2 = SpectralLibrary.readFrom(pathCSV)
+ except Exception as ex:
+ self.fail('Unable to read CSV. {}'.format(ex))
+
- self.assertEqual(len(sl),1)
- self.assertEqual(sl[0], self.SP)
self.SPECLIB = sl
diff --git a/timeseriesviewer/pixelloader.py b/timeseriesviewer/pixelloader.py
index d4edb1d8..528fd666 100644
--- a/timeseriesviewer/pixelloader.py
+++ b/timeseriesviewer/pixelloader.py
@@ -95,6 +95,18 @@ class PixelLoaderTask(object):
self.__dict__[k] = kwargs[k]
+ def validPixelValues(self, i):
+ if not self.success():
+ return False
+ if i >= len(self.resProfiles):
+ return False
+
+ profileData = self.resProfiles[i]
+ if profileData in [INFO_OUT_OF_IMAGE, INFO_NO_DATA]:
+ return False
+ else:
+ return True
+
def imageCrs(self):
return QgsCoordinateReferenceSystem(self.resCrsWkt)
@@ -365,23 +377,30 @@ def pixelLoadingLoop(inputQueue, resultQueue, cancelEvent, finishedEvent):
assert isinstance(finishedEvent, Event)
- while not cancelEvent.is_set():
-
- if not inputQueue.empty():
- print('Taskloop {} '.format(time.time()))
- task = inputQueue.get()
- if not isinstance(task, str):
- try:
- task = doLoaderTask(task)
- print('Taskloop put {} '.format(task))
- resultQueue.put(task)
- except Exception as ex:
- resultQueue.put(ex)
- elif task.startswith('LAST'):
- resultQueue.put('FINISHED')
- finishedEvent.set()
- else:
- print('Unhandled {}'.format(task))
+ while not inputQueue.empty():
+ if cancelEvent.is_set():
+ print('Taskloop put CANCELED')
+ break
+ #if not inputQueue.empty():
+ print('Taskloop {} '.format(time.time()))
+ task = inputQueue.get()
+ if not isinstance(task, str):
+ try:
+ print('Taskloop do {} '.format(task))
+ task = doLoaderTask(task)
+ print('Taskloop put {} '.format(task))
+ resultQueue.put(task)
+ except Exception as ex:
+ resultQueue.put(ex)
+ elif task.startswith('LAST'):
+ print('Taskloop put FINISHED')
+ resultQueue.put('FINISHED')
+ finishedEvent.set()
+ print('Taskloop FINISHED set')
+ else:
+ print('Taskloop put UNHANDLED')
+ print('Unhandled {}'.format(task))
+ break
resultQueue.put('CANCELED')
@@ -451,13 +470,18 @@ class PixelLoader(QObject):
self.mWorkerProcess = None
self.queueCheckTimer = QTimer() #
- #self.queueCheckTimer.setInterval(1000)
+ #self.queueCheckTimer.setInterval(200)
self.queueCheckTimer.timeout.connect(self.checkTaskResults)
- self.queueCheckTimer.start(200)
+ self.queueCheckTimer.timeout.connect(self.dummySlot)
+ self.queueCheckTimer.start(1000)
+
+ def dummySlot(self, *args):
+ print('dummy slot triggered')
+
def initWorkerProcess(self):
- if not isinstance(self.mWorkerProcess, multiprocessing.Process) or not self.mWorkerProcess.is_alive():
+ if not isinstance(self.mWorkerProcess, multiprocessing.Process) :
self.mWorkerProcess = multiprocessing.Process(name='PixelLoaderWorkingProcess',
target=pixelLoadingLoop,
args=(self.mTaskQueue, self.mResultQueue, self.mCancelEvent, self.mKillEvent,))
@@ -465,6 +489,10 @@ class PixelLoader(QObject):
self.mWorkerProcess.daemon = False
self.mWorkerProcess.start()
+ else:
+ if not self.mWorkerProcess.is_alive():
+ self.mWorkerProcess.run()
+
@@ -506,13 +534,13 @@ class PixelLoader(QObject):
self.jobProgress[jobId] = LoadingProgress(jobId, len(tasks))
self.sigLoadingStarted.emit(paths[:])
-
+ self.mKillEvent.clear()
self.initWorkerProcess()
for t in tasks:
assert isinstance(t, PixelLoaderTask)
t.mJobId = self.jobId
self.mTaskQueue.put(t)
- self.mKillEvent.clear()
+
self.mTaskQueue.put('LAST_{}'.format(jobId))
def cancelLoading(self):
@@ -526,34 +554,35 @@ class PixelLoader(QObject):
def checkTaskResults(self, *args):
dataList = []
-
+ print('CHECK TASK RESULTS')
finished = False
canceled = False
- while not self.mResultQueue.empty():
- data = self.mResultQueue.get()
- if isinstance(data, PixelLoaderTask):
- dataList.append(data)
- print('result pulled')
- elif isinstance(data, str):
- if data == 'FINISHED':
- finished = True
- elif data == 'CANCELED':
- canceled = True
+ if isinstance(self.mWorkerProcess, multiprocessing.Process):
+ while not self.mResultQueue.empty():
+ data = self.mResultQueue.get()
+ if isinstance(data, PixelLoaderTask):
+ dataList.append(data)
+ print('result pulled')
+ elif isinstance(data, str):
+ if data == 'FINISHED':
+ finished = True
+ elif data == 'CANCELED':
+ canceled = True
+ else:
+ s = ""
else:
- s = ""
- else:
- raise Exception('Unhandle type returned {}'.format(data))
- if len(dataList) > 0:
- self.onPixelLoaded(dataList)
+ raise Exception('Unhandled type returned {}'.format(data))
+ if len(dataList) > 0:
+ self.onPixelLoaded(dataList)
- if finished:
- dt = np.datetime64('now', 'ms') - self.mLoadingStartTime
- self.sigLoadingFinished.emit(dt)
+ if finished:
+ dt = np.datetime64('now', 'ms') - self.mLoadingStartTime
+ self.sigLoadingFinished.emit(dt)
- if self.mTaskQueue.empty() and self.mResultQueue.empty():
- self.mWorkerProcess.terminate()
- self.mWorkerProcess.join()
+ if self.mTaskQueue.empty() and self.mResultQueue.empty():
+ self.mWorkerProcess.terminate()
+ self.mWorkerProcess.join()
diff --git a/timeseriesviewer/profilevisualization.py b/timeseriesviewer/profilevisualization.py
index 4273c420..c91cd0cc 100644
--- a/timeseriesviewer/profilevisualization.py
+++ b/timeseriesviewer/profilevisualization.py
@@ -1560,8 +1560,8 @@ class SpectralTemporalVisualization(QObject):
if not isinstance(self.plotSettingsModel2D, PlotSettingsModel2D):
return False
- if not self.pixelLoader.isReadyToLoad():
- return False
+ #if not self.pixelLoader.isReadyToLoad():
+ # return False
assert isinstance(self.TS, TimeSeries)
@@ -1639,7 +1639,7 @@ class SpectralTemporalVisualization(QObject):
if len(tasks) > 0:
aGoodDefault = 2 if len(self.TS) > 25 else 1
- self.pixelLoader.setNumberOfProcesses(SETTINGS.value('profileloader_threads', aGoodDefault))
+ #self.pixelLoader.setNumberOfProcesses(SETTINGS.value('profileloader_threads', aGoodDefault))
if DEBUG:
print('Start loading for {} geometries from {} sources...'.format(
len(theGeometries), len(tasks)
diff --git a/timeseriesviewer/temporalprofiles.py b/timeseriesviewer/temporalprofiles.py
index 5370479d..9c42fb0c 100644
--- a/timeseriesviewer/temporalprofiles.py
+++ b/timeseriesviewer/temporalprofiles.py
@@ -650,23 +650,22 @@ class TemporalProfile(QObject):
i = d.temporalProfileIDs.index(self.mID)
tsd = self.mTimeSeries.getTSD(d.sourcePath)
assert isinstance(tsd, TimeSeriesDatum)
- profileData = d.resProfiles[i]
- if not isinstance(profileData, tuple):
- s = ""
- try:
+
+ if d.validPixelValues(i):
+
+ profileData = d.resProfiles[i]
+
vMean, vStd = profileData
- except Exception as ex:
- s = ""
- values = {}
- validValues = not isinstance(vMean, str)
- # 1. add the pixel values per returned band
-
- for iBand, bandIndex in enumerate(d.bandIndices):
- key = 'b{}'.format(bandIndex + 1)
- values[key] = vMean[iBand] if validValues else None
- key = 'std{}'.format(bandIndex + 1)
- values[key] = vStd[iBand] if validValues else None
- self.updateData(tsd, values)
+ values = {}
+ validValues = not isinstance(vMean, str)
+ # 1. add the pixel values per returned band
+
+ for iBand, bandIndex in enumerate(d.bandIndices):
+ key = 'b{}'.format(bandIndex + 1)
+ values[key] = vMean[iBand] if validValues else None
+ key = 'std{}'.format(bandIndex + 1)
+ values[key] = vStd[iBand] if validValues else None
+ self.updateData(tsd, values)
def loadMissingData(self, showGUI=False):
--
GitLab