-
benjamin.jakimow@geo.hu-berlin.de authoredbenjamin.jakimow@geo.hu-berlin.de authored
Code owners
Assign users and groups as approvers for specific file changes. Learn more.
make.py 19.75 KiB
# -*- coding: utf-8 -*-
import os, sys, fnmatch, six, subprocess, re
from qgis import *
from qgis.core import *
from qgis.gui import *
from PyQt5.QtGui import *
from PyQt5.QtCore import *
from PyQt5.QtSvg import *
from PyQt5.QtXml import *
from PyQt5.QtXmlPatterns import *
from PyQt5.uic.Compiler.qtproxies import QtGui
import gdal
from timeseriesviewer import DIR_UI, DIR_REPO, file_search
jp = os.path.join
def createFilePackage(dirData, recursive=True):
import numpy as np
from timeseriesviewer import DIR_REPO
pathInit = jp(dirData, '__init__.py')
code = ['#!/usr/bin/env python',
'"""',
'This file is auto-generated.',
'Do not edit manually, as changes might get overwritten.',
'"""',
'__author__ = "auto-generated by {}"'.format(os.path.relpath(__file__, DIR_REPO)),
'__date__ = "{}"'.format(np.datetime64('now')),
'',
'import sys, os',
'',
'thisDir = os.path.dirname(__file__)',
'# File path attributes:',
]
files = file_search(dirData, '*', recursive=recursive)
filePathAttributes = set()
def addFiles(files, comment=None, numberPrefix='File'):
if len(files) > 0:
if comment:
code.append('# '+comment)
for f in files:
an, ext = os.path.splitext(os.path.basename(f))
if re.search('^\d', an):
an = numberPrefix+an
an = re.sub(r'[-.]', '_',an)
assert an not in filePathAttributes
relpath = os.path.relpath(f, dirData)
code.append("{} = os.path.join(thisDir,r'{}')".format(an, relpath))
filePathAttributes.add(an)
code.append('\n')
raster = [f for f in files if re.search('.*\.(bsq|bip|bil|tif|tiff)$', f)]
vector = [f for f in files if re.search('.*\.(shp|kml|kmz)$', f)]
addFiles(raster, 'Raster files:', numberPrefix='Img_')
addFiles(vector, 'Vector files:', numberPrefix='Shp_')
#add self-test for file existence
if len(filePathAttributes) > 0:
code.extend(
[
"",
"# self-test to check each file path attribute",
"for a in dir(sys.modules[__name__]):",
" v = getattr(sys.modules[__name__], a)",
" if type(v) == str and os.path.isabs(v):" ,
" if not os.path.exists(v):",
" sys.stderr.write('Missing package attribute file: {}={}'.format(a, v))",
"",
"# cleanup",
"del thisDir, a, v ",
]
)
open(pathInit, 'w').write('\n'.join(code))
print('Created '+pathInit)
def getDOMAttributes(elem):
assert isinstance(elem, QDomElement)
values = dict()
attributes = elem.attributes()
for a in range(attributes.count()):
attr = attributes.item(a)
values[str(attr.nodeName())] = attr.nodeValue()
return values
def createTestData(dirTestData, pathTS, subsetRectangle, crs, drv=None):
lines = open(pathTS).readlines()
import tempfile, random
from timeseriesviewer.main import TimeSeries, TimeSeriesDatum
from qgis.core import QgsRectangle, QgsPoint, QgsPointV2, QgsCoordinateReferenceSystem
max_offset = 0 #in %
assert isinstance(subsetRectangle, QgsRectangle)
assert isinstance(crs, QgsCoordinateReferenceSystem)
TS = TimeSeries()
TS.loadFromFile(pathTS)
sw = subsetRectangle.width()
sh = subsetRectangle.height()
max_offset_x = sw / 100 * max_offset
max_offset_y = sw / 100 * max_offset
center = subsetRectangle.center()
if not os.path.exists(dirTestData):
os.mkdir(dirTestData)
dirImages = os.path.join(dirTestData, 'Images')
if not os.path.exists(dirImages):
os.mkdir(dirImages)
def random_offset():
offset_x = random.randrange(-max_offset_x, max_offset_x) if max_offset_x > 0 else 0
offset_y = random.randrange(-max_offset_y, max_offset_y) if max_offset_y > 0 else 0
return offset_x, offset_y
drvMEM = gdal.GetDriverByName('MEM')
from timeseriesviewer.main import transformGeometry
for TSD in TS.data:
assert isinstance(TSD, TimeSeriesDatum)
ox, oy = random_offset()
UL = QgsPoint(subsetRectangle.xMinimum() + ox,
subsetRectangle.yMaximum() + oy)
LR = QgsPoint(subsetRectangle.xMaximum() + ox,
subsetRectangle.yMinimum() + oy)
UL = transformGeometry(UL, crs, TSD.crs)
LR = transformGeometry(LR, crs, TSD.crs)
BBOX = QgsRectangle(UL, LR)
if not BBOX.intersects(TSD.getBoundingBox()):
print('Please note: no intersection with BBOX: '+TSD.pathImg)
#crop src dataset to BBOX
#for this we use GDAL
LUT_EXT = {'ENVI':'.bsq'}
filesToCopy = [f for f in [TSD.pathImg, TSD.pathMsk] if f is not None and os.path.exists(f)]
for pathSrc in filesToCopy:
dsSrc = gdal.Open(pathSrc)
assert isinstance(dsSrc, gdal.Dataset)
proj = dsSrc.GetProjection()
trans = list(dsSrc.GetGeoTransform())
trans[0] = UL.x()
trans[3] = UL.y()
nsDst = int(BBOX.width() / TSD.lyrImg.rasterUnitsPerPixelX())
nlDst = int(BBOX.height() / TSD.lyrImg.rasterUnitsPerPixelY())
dsDst = drvMEM.Create('', nsDst, nlDst, dsSrc.RasterCount, eType = dsSrc.GetRasterBand(1).DataType)
assert isinstance(dsDst, gdal.Dataset)
dsDst.SetProjection(proj)
dsDst.SetGeoTransform(trans)
wo = gdal.WarpOptions()
r = gdal.Warp(dsDst, dsSrc)
assert r > 0
drvDst = gdal.GetDriverByName(drv) if drv is not None else dsSrc.GetDriver()
#try to retireve an extension
pathDst = os.path.join(dirImages, os.path.splitext(os.path.basename(pathSrc))[0])
ext = drvDst.GetMetadata_Dict().get('DMD_EXTENSION','')
if ext == '':
ext = LUT_EXT.get(drvDst.ShortName, '')
if not pathDst.endswith(ext):
pathDst += ext
print('Write {}'.format(pathDst))
drvDst.CreateCopy(pathDst, dsDst)
def compile_rc_files(ROOT, removeTimeStamp=True):
#find ui files
ui_files = file_search(ROOT, '*.ui', recursive=True)
qrcs = set()
doc = QDomDocument()
reg = re.compile('(?<=resource=")[^"]+\.qrc(?=")')
for ui_file in ui_files:
pathDir = os.path.dirname(ui_file)
doc.setContent(QFile(ui_file))
includeNodes = doc.elementsByTagName('include')
for i in range(includeNodes.count()):
attr = getDOMAttributes(includeNodes.item(i).toElement())
if 'location' in attr.keys():
print((ui_file, str(attr['location'])))
qrcs.add((pathDir, str(attr['location']), ui_file))
#compile Qt resource files
#resourcefiles = file_search(ROOT, '*.qrc', recursive=True)
resourcefiles = list(qrcs)
assert len(resourcefiles) > 0
if sys.platform == 'darwin':
prefix = '/Applications/QGIS.app/Contents/MacOS/bin/'
else:
prefix = ''
for root_dir, f, ui_file in resourcefiles:
#dn = os.path.dirname(f)
pathQrc = os.path.normpath(jp(root_dir, f))
if not os.path.exists(pathQrc):
raise Exception('Missing: {} \n used in {}'.format(pathQrc, ui_file))
bn = os.path.basename(f)
bn = os.path.splitext(bn)[0]
pathPy2 = os.path.join(DIR_UI, bn+'.py' )
subprocess.call(['pyrcc4', '-py2', '-o', pathPy2, pathQrc])
if removeTimeStamp:
lines = open(pathPy2).readlines()
lines = re.sub('# Created: .*\n', "# Created <timestamp removed>\n", ''.join(lines))
f = open(pathPy2, 'w')
f.write(lines)
f.flush()
f.close()
def fileNeedsUpdate(file1, file2):
if not os.path.exists(file2):
return True
else:
if not os.path.exists(file1):
return True
else:
return os.path.getmtime(file1) > os.path.getmtime(file2)
def svg2png(pathDir, overwrite=False, mode='INKSCAPE', filterFile=None):
"""
Converts SVG files into PNG raster images
:param pathDir:
:param overwrite:
:param mode:
:return:
"""
assert mode in ['INKSCAPE', 'WEBKIT', 'SVG']
from PyQt5.QtWebKit import QWebPage
svgs = file_search(pathDir, '*.svg')
if filterFile is not None:
print('use file filter')
lines = open(filterFile).readlines()
lines = [l.strip() for l in lines]
lines = [l for l in lines if not l.startswith('#') or len(l) == 0]
svgs = [f for f in svgs if
f in lines or os.path.basename(f) in lines]
if len(svgs) == 0:
print('No SVGs to convert')
return
app = QApplication([], True)
buggySvg = []
for pathSvg in svgs:
dn = os.path.dirname(pathSvg)
bn, _ = os.path.splitext(os.path.basename(pathSvg))
pathPng = jp(dn, bn+'.png')
if mode == 'SVG':
renderer = QSvgRenderer(pathSvg)
doc_size = renderer.defaultSize() # size in px
img = QImage(doc_size, QImage.Format_ARGB32)
#img.fill(0xaaA08080)
painter = QPainter(img)
renderer.render(painter)
painter.end()
if overwrite or not os.path.exists(pathPng):
img.save(pathPng, quality=100)
del painter, renderer
elif mode == 'WEBKIT':
page = QWebPage()
frame = page.mainFrame()
f = QFile(pathSvg)
if f.open(QFile.ReadOnly | QFile.Text):
textStream = QTextStream(f)
svgData = textStream.readAll()
f.close()
qba = QByteArray(str(svgData))
frame.setContent(qba,"image/svg+xml")
page.setViewportSize(frame.contentsSize())
palette = page.palette()
background_color = QColor(50,0,0,50)
palette.setColor(QPalette.Window, background_color)
brush = QBrush(background_color)
palette.setBrush(QPalette.Window, brush)
page.setPalette(palette)
img = QImage(page.viewportSize(), QImage.Format_ARGB32)
img.fill(background_color) #set transparent background
painter = QPainter(img)
painter.setBackgroundMode(Qt.OpaqueMode)
#print(frame.renderTreeDump())
frame.render(painter)
painter.end()
if overwrite or not os.path.exists(pathPng):
print('Save {}...'.format(pathPng))
img.save(pathPng, quality=100)
del painter, frame, img, page
s =""
elif mode == 'INKSCAPE':
if fileNeedsUpdate(pathSvg, pathPng):
if sys.platform == 'darwin':
cmd = ['inkscape']
else:
dirInkscape = r'C:\Program Files\Inkscape'
assert os.path.isdir(dirInkscape)
cmd = [jp(dirInkscape,'inkscape')]
cmd.append('--file={}'.format(pathSvg))
cmd.append('--export-png={}'.format(pathPng))
from subprocess import PIPE
p = subprocess.Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE)
output, err = p.communicate()
rc = p.returncode
print('Saved {}'.format(pathPng))
if err != '':
buggySvg.append((pathSvg, err))
if len(buggySvg) > 0:
six._print('SVG Errors')
for t in buggySvg:
pathSvg, error = t
six._print(pathSvg, error, file=sys.stderr)
s = ""
def file2qrc(icondir, pathQrc, qrcPrefix='timeseriesviewer', fileExtension='.png'):
pathQrc = os.path.abspath(pathQrc)
dirQrc = os.path.dirname(pathQrc)
app = QApplication([])
assert os.path.exists(pathQrc)
#create the new RCC
doc = QDomDocument('RCC')
doc.setContent(QFile(pathQrc))
if str(doc.toString()) == '':
doc.appendChild(doc.createElement('RCC'))
root = doc.documentElement()
fileOfInterest = set()
fileAttributes = {}
#add files that are already included in the QRC file
fileNodes = doc.elementsByTagName('file')
for i in range(fileNodes.count()):
fileNode = fileNodes.item(i).toElement()
file = str(fileNode.childNodes().item(0).nodeValue())
#if file.lower().endswith(fileExtension):
fileOfInterest.add(file)
if fileNode.hasAttributes():
attributes = {}
for i in range(fileNode.attributes().count()):
attr = fileNode.attributes().item(i).toAttr()
attributes[str(attr.name())] = str(attr.value())
fileAttributes[file] = attributes
#add new files from the icondir
for f in file_search(icondir, '*'+fileExtension):
file = os.path.relpath(f, dirQrc).replace('\\','/')
fileOfInterest.add(file)
fileOfInterest = sorted(list(fileOfInterest))
def elementsByTagAndProperties(elementName, attributeProperties, rootNode=None):
assert isinstance(elementName, str)
assert isinstance(attributeProperties, dict)
if rootNode is None:
rootNode = doc
resourceNodes = rootNode.elementsByTagName(elementName)
nodeList = []
for i in range(resourceNodes.count()):
resourceNode = resourceNodes.item(i).toElement()
for aName, aValue in attributeProperties.items():
if resourceNode.hasAttribute(aName):
if aValue != None:
assert isinstance(aValue, str)
if str(resourceNode.attribute(aName)) == aValue:
nodeList.append(resourceNode)
else:
nodeList.append(resourceNode)
return nodeList
resourceNodes = elementsByTagAndProperties('qresource', {'prefix':qrcPrefix})
if len(resourceNodes) == 0:
resourceNode = doc.createElement('qresource')
root.appendChild(resourceNode)
resourceNode.setAttribute('prefix', qrcPrefix)
elif len(resourceNodes) == 1:
resourceNode = resourceNodes[0]
else:
raise NotImplementedError('Multiple resource nodes')
#remove childs, as we have all stored in list pngFiles
childs = resourceNode.childNodes()
while not childs.isEmpty():
node = childs.item(0)
node.parentNode().removeChild(node)
#insert new childs
for file in fileOfInterest:
node = doc.createElement('file')
attributes = fileAttributes.get(file)
if attributes:
for k, v in attributes.items():
node.setAttribute(k,v)
s = 2
node.appendChild(doc.createTextNode(file))
resourceNode.appendChild(node)
print(file)
f = open(pathQrc, "w")
f.write(doc.toString())
f.close()
def updateInfoHTML():
import markdown, urllib
import timeseriesviewer
from timeseriesviewer import DIR_REPO, PATH_LICENSE, PATH_CHANGELOG
"""
Keyword arguments:
* input: a file name or readable object.
* output: a file name or writable object.
* encoding: Encoding of input and output.
* Any arguments accepted by the Markdown class.
"""
markdownExtension = [
'markdown.extensions.toc',
'markdown.extensions.tables',
'markdown.extensions.extra'
]
def readUrlTxt(url):
req = urllib.urlopen(url)
enc = req.headers['content-type'].split('charset=')[-1]
txt = req.read()
req.close()
if enc == 'text/plain':
return unicode(txt)
return unicode(txt, enc)
paths = [jp(DIR_REPO, *['LICENSE.md']),
jp(DIR_REPO, *['CHANGES.md'])]
for pathSrc in paths:
pathDst = pathSrc.replace('.md','.html')
markdown.markdownFromFile(input=pathSrc,
extensions=markdownExtension,
output=pathDst, output_format='html5')
def updateMetadataTxt():
#see http://docs.qgis.org/testing/en/docs/pyqgis_developer_cookbook/plugins.html#plugin-metadata
#for required metatags
pathDst = jp(DIR_REPO, 'metadata.txt')
assert os.path.exists(pathDst)
import timeseriesviewer.sandbox
#required to use QIcons
qgis = timeseriesviewer.sandbox.initQgisEnvironment()
import timeseriesviewer, collections
md = collections.OrderedDict()
for line in open(pathDst).readlines():
parts = line.split('=')
if len(parts) >= 2:
md[parts[0]] = '='.join(parts[1:])
#update/set new metadata
md['name'] = timeseriesviewer.TITLE
md['qgisMinimumVersion'] = "2.18"
#md['qgisMaximumVersion'] =
md['description'] = timeseriesviewer.DESCRIPTION.strip()
md['about'] = timeseriesviewer.ABOUT.strip()
md['version'] = timeseriesviewer.VERSION.strip()
md['author'] = "Benjamin Jakimow, Geomatics Lab, Humboldt-Universität zu Berlin"
md['email'] = "benjamin.jakimow@geo.hu-berlin.de"
#md['changelog'] =
md['experimental'] = "False"
md['deprecated'] = "False"
md['tags'] = "remote sensing, raster, time series"
md['homepage'] = timeseriesviewer.WEBSITE
md['repository'] = timeseriesviewer.WEBSITE
md['tracker'] = timeseriesviewer.WEBSITE+'/issues'
md['icon'] = r'timeseriesviewer/ui/icons/icon.png'
md['category'] = 'Raster'
lines = ['[general]']
for k, line in md.items():
lines.append('{}={}'.format(k, line))
open(pathDst, 'w').writelines('\n'.join(lines))
s = ""
def make_pb_tool_cfg():
pathPBToolCgf = r''
lines = open(pathPBToolCgf).readlines()
#main_dialog:
if __name__ == '__main__':
icondir = jp(DIR_UI, *['icons'])
pathQrc = jp(DIR_UI,'resources.qrc')
from timeseriesviewer import DIR_EXAMPLES
if False:
from qgis import *
from qgis.core import *
from qgis.gui import *
if sys.platform == 'darwin':
PATH_QGS = r'/Applications/QGIS.app/Contents/MacOS'
os.environ['GDAL_DATA'] = r'/usr/local/Cellar/gdal/1.11.3_1/share'
else:
# assume OSGeo4W startup
PATH_QGS = os.environ['QGIS_PREFIX_PATH']
assert os.path.exists(PATH_QGS)
qgsApp = QgsApplication([], True)
QApplication.addLibraryPath(r'/Applications/QGIS.app/Contents/PlugIns')
QApplication.addLibraryPath(r'/Applications/QGIS.app/Contents/PlugIns/qgis')
qgsApp.setPrefixPath(PATH_QGS, True)
qgsApp.initQgis()
pathDirTestData = os.path.join(DIR_EXAMPLES,'Images')
pathTS = r'C:\Users\geo_beja\Repositories\QGIS_Plugins\SenseCarbonTSViewer\make\testdata_sources.txt'
from qgis.core import QgsCoordinateReferenceSystem, QgsPoint, QgsRectangle
subset = QgsRectangle(QgsPoint(-55.36091,-6.79851), #UL
QgsPoint(-55.34132,-6.80514)) #LR
crs = QgsCoordinateReferenceSystem('EPSG:4326') # lat lon coordinates
createTestData(pathDirTestData, pathTS,subset, crs, drv='ENVI')
exit(0)
if False:
# update __init__.py of testdata directories
d = pathDirTestData = os.path.join(DIR_EXAMPLES,'Images')
#d = pathDirTestData = DIR_EXAMPLES
createFilePackage(d, recursive=False)
if False:
updateInfoHTML()
if False:
updateMetadataTxt()
if False:
# convert SVG to PNG
svg2png(icondir, overwrite=False,
filterFile=os.path.join(os.path.dirname(__file__), 'svg2png.txt'))
if False:
#add png icons to qrc file
#file2qrc(icondir, pathQrc, qrcPrefix='timeseriesviewer', fileExtension='.png')
file2qrc(icondir, pathQrc, qrcPrefix='timeseriesviewer', fileExtension='.svg')
if True:
compile_rc_files(DIR_UI)
print('Done')