Subversion Repositories basico

Rev

Rev 357 | Rev 359 | Go to most recent revision | Details | Compare with Previous | Last modification | View Log | RSS feed

Rev Author Line No. Line
340 t00mlabs 1
#!/usr/bin/python
2
# -*- coding: utf-8 -*-
3
"""
4
# File: srv_utils.py
5
# Author: Tomás Vírseda
6
# License: GPL v3
7
# Description: Generic functions service
8
"""
9
 
356 t00mlabs 10
import os
340 t00mlabs 11
import re
12
import sys
355 t00mlabs 13
import json
357 t00mlabs 14
from html import escape
356 t00mlabs 15
from stat import ST_SIZE
340 t00mlabs 16
import subprocess
17
import tarfile
18
import zipfile
19
import shutil
20
import requests
21
import webbrowser
22
import feedparser
23
from datetime import datetime
355 t00mlabs 24
from basico.core.mod_env import GPATH, LPATH, FILE
340 t00mlabs 25
from basico.core.mod_srv import Service
26
 
27
class Utils(Service):
28
    """
29
    Missing class docstring (missing-docstring)
30
    """
31
 
32
    def initialize(self):
33
        """
34
        Missing method docstring (missing-docstring)
35
        """
36
        self.get_services()
355 t00mlabs 37
        self.load_extensions()
340 t00mlabs 38
 
39
 
356 t00mlabs 40
    def get_services(self):
41
        """
42
        Missing method docstring (missing-docstring)
43
        """
44
        self.srvdtb = self.get_service('DB')
45
 
46
 
355 t00mlabs 47
    def load_extensions(self):
48
        with open(FILE['EXT'], 'r') as fext:
49
            self.extensions = json.load(fext)
50
            self.log.debug("%d extensions loaded", len(self.extensions))
51
 
52
 
356 t00mlabs 53
    def get_file_metadata(self, path):
54
        self.log.debug("Getting metadata from: %s", path)
55
        metadata = {}
357 t00mlabs 56
        metadata['Title'] = escape(self.get_file_name_with_ext(path))
57
        metadata['Basename'] = escape(self.get_file_basename(path))
58
        metadata['Extension'] = escape(self.get_file_extension(path))
356 t00mlabs 59
        metadata['Size'] = self.get_file_size(path)
357 t00mlabs 60
        metadata['Description'] = escape(self.get_file_description(metadata['Extension']))
61
        metadata['Mimetype'] = escape(self.get_file_mimetype(metadata['Extension']))
62
        metadata['Doctype'] = escape(self.get_file_doctype(metadata['Mimetype']))
356 t00mlabs 63
        return metadata
64
 
340 t00mlabs 65
 
356 t00mlabs 66
    def get_file_size(self, path):
67
        # Get size in bytes
68
        size = os.stat(path)[ST_SIZE]
69
        self.log.debug("\tSize: %s", str(size))
70
        return size
340 t00mlabs 71
 
356 t00mlabs 72
 
73
    def get_file_extension(self, path):
74
        # Get extension
75
        rest, extension = os.path.splitext(path)
76
        ext = (extension[1:]).lower()
358 t00mlabs 77
        ext = ext.strip()
78
        if len(ext) > 0:
356 t00mlabs 79
            return '%s' % ext
80
        else:
81
            return '#noext#'
82
 
83
 
84
    def get_file_basename(self, path):
85
        # Get basename
86
        rest, extension = os.path.splitext(path)
87
        basename = os.path.basename(rest)
88
        self.log.debug("\tBasename: %s", basename)
89
        return basename
90
 
91
 
92
    def get_file_name_with_ext(self, path):
93
        # Filename
94
        basename = self.get_file_basename(path)
95
        extension = self.get_file_extension(path)
96
        if extension == '#noext#':
358 t00mlabs 97
            title = basename
98
        else:
99
            title = '%s.%s' % (basename, extension)
356 t00mlabs 100
        self.log.debug("\tTitle: %s", title)
101
        return title
102
 
103
 
104
    def get_file_mimetype(self, ext):
105
        try:
106
            mimetype, description = self.extensions[ext]
107
            self.log.debug("\tMimetype: %s", mimetype)
108
            return mimetype
358 t00mlabs 109
        except Exception as warning:
110
            self.log.warning(warning)
111
            return 'application/x-other'
356 t00mlabs 112
 
113
 
114
    def get_file_doctype(self, mimetype):
115
        mtype = mimetype[:mimetype.rfind('/')]
116
        doctype = mtype.title()
117
        self.log.debug("\tDoctype: %s", doctype)
118
        return doctype
119
 
120
 
355 t00mlabs 121
    def get_file_description(self, ext):
122
        try:
123
            filetype, description = self.extensions[ext]
356 t00mlabs 124
            self.log.debug("\tDescription: %s", description)
355 t00mlabs 125
            return description
358 t00mlabs 126
        except Exception as warning:
127
            self.log.error(warning)
128
            return 'Unknow file type'
355 t00mlabs 129
 
130
 
353 t00mlabs 131
    def get_disk_usage(self, path):
132
        """
133
        Get disk usage for a given path
134
        """
135
        return shutil.disk_usage(path)
136
 
137
 
138
    def get_disk_usage_fraction(self, path):
139
        """
140
        Get disk usage as a fraction for using with Gtk.Progressbar)
141
        """
142
        total, used, free = self.get_disk_usage(path)
143
        return used/total
144
 
356 t00mlabs 145
 
146
    def get_kilobytes(self, integer_bytes):
147
        kilo, bites = divmod(int(abs(integer_bytes)), 1_024)
148
        # ~ self.log.debug("%d bytes -> %d Kb", integer_bytes, kilo)
149
        return kilo
150
 
151
 
152
    def get_megabytes(self, integer_bytes):
153
        kilo = self.get_kilobytes(integer_bytes)
154
        mega, kilo = divmod(kilo, 1_024)
155
        # ~ self.log.debug("%d bytes -> %d Mb", integer_bytes, mega)
156
        return mega
157
 
158
 
159
    def get_gigabytes(self, integer_bytes):
160
        mega = self.get_megabytes(integer_bytes)
161
        giga, mega = divmod(mega, 1_024)
162
        # ~ self.log.debug("%d bytes -> %d Gb", integer_bytes, giga)
163
        return giga
164
 
165
 
166
    def get_human_sizes(self, integer_bytes):
167
        if integer_bytes < 1024:
168
            return "%d bytes" % integer_bytes
169
        elif integer_bytes > 1024 and integer_bytes < (1024*1024):
170
            return "%d Kb" % self.get_kilobytes(integer_bytes)
171
        elif integer_bytes > 1024*1024 and integer_bytes < (1024*1024*1024):
172
            return "%d Mb" % self.get_megabytes(integer_bytes)
173
        elif integer_bytes > 1024*1024*1024 and integer_bytes < (1024*1024*1024*1024):
174
            return "%d Gb" % self.get_gigabytes(integer_bytes)
175
 
176
 
353 t00mlabs 177
    def get_disk_usage_human(self, path):
178
        """
179
        Get disk usage as a fraction for using with Gtk.Progressbar)
180
        Some bytes borrowed from:
181
        https://github.com/juancarlospaco/anglerfish/blob/master/anglerfish/bytes2human.py
356 t00mlabs 182
        """            
353 t00mlabs 183
        total, used, free = self.get_disk_usage(path)
356 t00mlabs 184
        tgb = self.get_gigabytes(total)
185
        ugb = self.get_gigabytes(used)
186
        fgb = self.get_gigabytes(total-used)
353 t00mlabs 187
        humansize = "Using %dGb of %dGb (Free space: %dGb)" % (int(ugb), int(tgb), int(fgb))
188
        return humansize
189
 
340 t00mlabs 190
    def timestamp(self):
191
        """
192
        Missing method docstring (missing-docstring)
193
        """
194
        now = datetime.now()
195
        timestamp = "%4d%02d%02d_%02d%02d%02d" % (now.year, now.month, now.day, now.hour, now.minute, now.second)
196
 
197
        return timestamp
198
 
199
 
200
    def get_datetime(self, timestamp):
201
        """
202
        Missing method docstring (missing-docstring)
203
        """
204
        adate = datetime.strptime(timestamp, "%Y%m%d_%H%M%S")
205
        return adate
206
 
207
 
208
    def get_human_date_from_timestamp(self, timestamp=None):
209
        """
210
        Missing method docstring (missing-docstring)
211
        """
212
        if timestamp is None:
213
            timestamp = self.timestamp()
214
 
215
        adate = self.get_datetime(timestamp)
216
        return "%s" % adate.strftime("%Y/%m/%d %H:%M")
217
 
218
 
219
    def get_excel_date(self, timestamp):
220
        timestamp = timestamp.replace('-', '/')        
221
        timestamp = timestamp.replace('T', ' ')
222
        adate = datetime.strptime(timestamp, "%Y/%m/%d %H:%M:%S")
223
        excel_date = "%s" % adate.strftime("%d/%m/%Y")
224
        return excel_date
225
 
226
 
227
    def fuzzy_date_from_timestamp(self, timestamp):
228
        """
229
        Missing method docstring (missing-docstring)
230
        """
231
        d1 = self.get_datetime(timestamp)
232
        d2 = datetime.now()
233
        rdate = d2 - d1 # DateTimeDelta
234
        if rdate.days > 0:
235
            if rdate.days <= 31:
236
                return "%d days ago" % int(rdate.days)
237
 
238
            if rdate.days > 31 and rdate.days < 365:
239
                return "%d months ago" % int((rdate.days/31))
240
 
241
            if rdate.days >= 365:
242
                return "%d years ago" % int((rdate.days/365))
243
 
244
        hours = rdate.seconds / 3600
245
        if int(hours) > 0:
246
            return "%d hours ago" % int(hours)
247
 
248
        minutes = rdate.seconds / 60
249
        if int(minutes) > 0:
250
            return "%d minutes ago" % int(minutes)
251
 
252
        if int(rdate.seconds) > 0:
253
            return "%d seconds ago" % int(rdate.seconds)
254
 
255
        if int(rdate.seconds) == 0:
256
            return "Right now"
257
 
258
 
259
    def browse(self, url):
260
        """
261
        Missing method docstring (missing-docstring)
262
        """
263
        if sys.platform in ['linux', 'linux2']:
264
            browser = webbrowser.get('firefox')
265
        elif sys.platform == 'win32':
266
            browser = webbrowser.get('windows-default')
267
 
268
        browser.open_new_tab(url)
269
 
270
 
271
    def which(self, program):
272
        """
273
        Missing method docstring (missing-docstring)
274
        """
275
        if sys.platform == 'win32':
276
            program = program + '.exe'
277
 
278
        def is_exe(fpath):
279
            """
280
            Missing method docstring (missing-docstring)
281
            """
282
            return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
283
 
284
        fpath, fname = os.path.split(program)
285
        if fpath:
286
            if is_exe(program):
287
                return program
288
        else:
289
            for path in os.environ["PATH"].split(os.pathsep):
290
                path = path.strip('"')
291
                exe_file = os.path.join(path, program)
292
                if is_exe(exe_file):
293
                    return exe_file
294
 
295
        return None
296
 
297
 
298
    def install_geckodriver(self):
299
        """
300
        Install Gecko Driver
301
        """
302
        GECKODRIVER_LINUX = GPATH['DRIVERS'] + 'geckodriver'
303
        GECKODRIVER_WIN32 = GPATH['DRIVERS'] + 'geckodriver.exe'
304
        GECKO_INSTALL_DIR = LPATH['DRIVERS']
305
 
306
        if sys.platform == 'linux':
307
            shutil.copy(GECKODRIVER_LINUX, GECKO_INSTALL_DIR)
308
        else:
309
            shutil.copy(GECKODRIVER_WIN32, GECKO_INSTALL_DIR)
310
        self.log.debug("Gecko driver installed to %s" % GECKO_INSTALL_DIR)
311
 
312
 
313
    def download(self, prgname, source, target):
314
        """
315
        Missing method docstring (missing-docstring)
316
        """
317
        try:
318
            self.log.debug("Downloading %s from: %s" % (prgname, source))
319
            response = requests.get(source, stream=True)
320
            with open(target, 'wb') as out_file:
321
                shutil.copyfileobj(response.raw, out_file)
322
            del response
323
            self.log.debug("%s downloaded to %s" % (prgname, target))
324
            return True
325
        except Exception as error:
326
            self.log.error(error)
327
            return False
328
 
329
 
330
    def extract(self, filename, target_path, protocol):
331
        """
332
        Missing method docstring (missing-docstring)
333
        """
334
        self.log.debug("Extracting %s to %s using protocol %s" % (filename, target_path, protocol))
335
        if protocol in ['tar.gz', 'bz2']:
336
            try:
337
                tar = tarfile.open(filename, "r:*")
338
                tar.extractall(target_path)
339
                tar.close()
340
                self.log.debug("Extracted successfully")
341
                return True
342
            except Exception as error:
343
                self.log.error(error)
344
                return False
345
        elif protocol == 'zip':
346
            try:
347
                self.unzip(filename, target_path)
348
                self.log.debug("Extracted successfully")
349
                return True
350
            except Exception as error:
351
                self.log.error(error)
352
                return False
353
 
354
    def zip(self, filename, directory):
355
        """
356
        Zip directory and rename to .bco
357
        """
358
        # http://stackoverflow.com/a/25650295
359
        #~ make_archive(archive_name, 'gztar', root_dir)
360
        self.log.debug("Target: %s", filename)
361
        sourcename = os.path.basename(filename)
362
        dot = sourcename.find('.')
363
        if dot == -1:
364
            basename = sourcename
365
        else:
366
            basename = sourcename[:dot]
367
        sourcedir = os.path.dirname(filename)
368
        source = os.path.join(sourcedir, basename)
369
        zipfile = shutil.make_archive(source, 'zip', directory)
370
        target = source + '.bco'
371
        shutil.move(zipfile, target)
372
        return target
373
 
374
 
375
    def unzip(self, target, install_dir):
376
        """
377
        Unzip file to a given dir
378
        """
379
        zip_archive = zipfile.ZipFile(target, "r")
380
        zip_archive.extractall(path=install_dir)
381
        zip_archive.close()
382
 
383
 
384
    def get_firefox_profile_dir(self):
385
        """
386
        Self-explained. Get default Firefox directory
387
        """
388
        if sys.platform in ['linux', 'linux2']:
389
            cmd = "ls -d /home/$USER/.mozilla/firefox/*.default/"
390
            p = subprocess.Popen([cmd], shell=True, stdout=subprocess.PIPE)
391
            FF_PRF_DIR = p.communicate()[0][0:-2]
392
            FF_PRF_DIR_DEFAULT = str(FF_PRF_DIR, 'utf-8')
393
        elif sys.platform == 'win32':
394
            import glob
395
            APPDATA = os.getenv('APPDATA')
396
            FF_PRF_DIR = "%s\\Mozilla\\Firefox\\Profiles\\" % APPDATA
397
            PATTERN = FF_PRF_DIR + "*default*"
398
            FF_PRF_DIR_DEFAULT = glob.glob(PATTERN)[0]
399
 
400
        return FF_PRF_DIR_DEFAULT
401
 
402
 
403
    def feedparser_parse(self, thing):
404
        """
405
        Missing method docstring (missing-docstring)
406
        """
407
        try:
408
            return feedparser.parse(thing)
409
        except TypeError:
410
            if 'drv_libxml2' in feedparser.PREFERRED_XML_PARSERS:
411
                feedparser.PREFERRED_XML_PARSERS.remove('drv_libxml2')
412
                return feedparser.parse(thing)
413
            else:
414
                self.log.error(self.get_traceback())
415
                return None
416
 
417
 
418
    def check(self):
419
        """
420
        Check Basico environment
421
        """
422
        gtk_version = self.uif.check_gtk_version() # Check GTK version
423
        gecko_driver = self.driver.check() # Check Gecko webdrver
424
        run = gtk_version and gecko_driver
425
        if run:
426
            self.log.debug("Basico environment ready!")
427
        else:
428
            self.log.error("Error(s) found checking Basico environment")
429
 
430
        return run
431
 
432
 
433
 
434
 
435
    def clean_html(self, raw_html):
436
        cleanr = re.compile('<.*?>')
437
        cleantext = re.sub(cleanr, '', raw_html)
438
        return cleantext