Subversion Repositories basico

Rev

Rev 355 | Rev 357 | Go to most recent revision | Details | Compare with Previous | Last modification | View Log | RSS feed

Rev Author Line No. Line
340 t00mlabs 1
#!/usr/bin/python
2
# -*- coding: utf-8 -*-
3
"""
4
# File: srv_utils.py
5
# Author: Tomás Vírseda
6
# License: GPL v3
7
# Description: Generic functions service
8
"""
9
 
356 t00mlabs 10
import os
340 t00mlabs 11
import re
12
import sys
355 t00mlabs 13
import json
356 t00mlabs 14
from stat import ST_SIZE
340 t00mlabs 15
import subprocess
16
import tarfile
17
import zipfile
18
import shutil
19
import requests
20
import webbrowser
21
import feedparser
22
from datetime import datetime
355 t00mlabs 23
from basico.core.mod_env import GPATH, LPATH, FILE
340 t00mlabs 24
from basico.core.mod_srv import Service
25
 
26
class Utils(Service):
27
    """
28
    Missing class docstring (missing-docstring)
29
    """
30
 
31
    def initialize(self):
32
        """
33
        Missing method docstring (missing-docstring)
34
        """
35
        self.get_services()
355 t00mlabs 36
        self.load_extensions()
340 t00mlabs 37
 
38
 
356 t00mlabs 39
    def get_services(self):
40
        """
41
        Missing method docstring (missing-docstring)
42
        """
43
        self.srvdtb = self.get_service('DB')
44
 
45
 
355 t00mlabs 46
    def load_extensions(self):
47
        with open(FILE['EXT'], 'r') as fext:
48
            self.extensions = json.load(fext)
49
            self.log.debug("%d extensions loaded", len(self.extensions))
50
 
51
 
356 t00mlabs 52
    def get_file_metadata(self, path):
53
        self.log.debug("Getting metadata from: %s", path)
54
        metadata = {}
55
        metadata['Title'] = self.get_file_name_with_ext(path)
56
        metadata['Basename'] = self.get_file_basename(path)
57
        metadata['Extension'] = self.get_file_extension(path)
58
        metadata['Size'] = self.get_file_size(path)
59
        metadata['Description'] = self.get_file_description(metadata['Extension'])
60
        metadata['Mimetype'] = self.get_file_mimetype(metadata['Extension'])
61
        metadata['Doctype'] = self.get_file_doctype(metadata['Mimetype'])
62
        return metadata
63
 
340 t00mlabs 64
 
356 t00mlabs 65
    def get_file_size(self, path):
66
        # Get size in bytes
67
        size = os.stat(path)[ST_SIZE]
68
        self.log.debug("\tSize: %s", str(size))
69
        return size
340 t00mlabs 70
 
356 t00mlabs 71
 
72
    def get_file_extension(self, path):
73
        # Get extension
74
        rest, extension = os.path.splitext(path)
75
        ext = (extension[1:]).lower()
76
        self.log.debug("\tExtension: %s", ext)
77
        if (len(ext) != 0):
78
            return '%s' % ext
79
        else:
80
            return '#noext#'
81
 
82
 
83
    def get_file_basename(self, path):
84
        # Get basename
85
        rest, extension = os.path.splitext(path)
86
        basename = os.path.basename(rest)
87
        self.log.debug("\tBasename: %s", basename)
88
        return basename
89
 
90
 
91
    def get_file_name_with_ext(self, path):
92
        # Filename
93
        basename = self.get_file_basename(path)
94
        extension = self.get_file_extension(path)
95
        if extension == '#noext#':
96
            extension = ''
97
        title = '%s.%s' % (basename, extension)
98
        self.log.debug("\tTitle: %s", title)
99
        return title
100
 
101
 
102
    def get_file_mimetype(self, ext):
103
        try:
104
            mimetype, description = self.extensions[ext]
105
            self.log.debug("\tMimetype: %s", mimetype)
106
            return mimetype
107
        except Exception as error:
108
            self.log.error(error)
109
            return None
110
 
111
 
112
 
113
    def get_file_doctype(self, mimetype):
114
        mtype = mimetype[:mimetype.rfind('/')]
115
        doctype = mtype.title()
116
        self.log.debug("\tDoctype: %s", doctype)
117
        return doctype
118
 
119
 
355 t00mlabs 120
    def get_file_description(self, ext):
121
        try:
122
            filetype, description = self.extensions[ext]
356 t00mlabs 123
            self.log.debug("\tDescription: %s", description)
355 t00mlabs 124
            return description
125
        except:
126
            return None
127
 
128
 
353 t00mlabs 129
    def get_disk_usage(self, path):
130
        """
131
        Get disk usage for a given path
132
        """
133
        return shutil.disk_usage(path)
134
 
135
 
136
    def get_disk_usage_fraction(self, path):
137
        """
138
        Get disk usage as a fraction for using with Gtk.Progressbar)
139
        """
140
        total, used, free = self.get_disk_usage(path)
141
        return used/total
142
 
356 t00mlabs 143
 
144
    def get_kilobytes(self, integer_bytes):
145
        kilo, bites = divmod(int(abs(integer_bytes)), 1_024)
146
        # ~ self.log.debug("%d bytes -> %d Kb", integer_bytes, kilo)
147
        return kilo
148
 
149
 
150
    def get_megabytes(self, integer_bytes):
151
        kilo = self.get_kilobytes(integer_bytes)
152
        mega, kilo = divmod(kilo, 1_024)
153
        # ~ self.log.debug("%d bytes -> %d Mb", integer_bytes, mega)
154
        return mega
155
 
156
 
157
    def get_gigabytes(self, integer_bytes):
158
        mega = self.get_megabytes(integer_bytes)
159
        giga, mega = divmod(mega, 1_024)
160
        # ~ self.log.debug("%d bytes -> %d Gb", integer_bytes, giga)
161
        return giga
162
 
163
 
164
    def get_human_sizes(self, integer_bytes):
165
        if integer_bytes < 1024:
166
            return "%d bytes" % integer_bytes
167
        elif integer_bytes > 1024 and integer_bytes < (1024*1024):
168
            return "%d Kb" % self.get_kilobytes(integer_bytes)
169
        elif integer_bytes > 1024*1024 and integer_bytes < (1024*1024*1024):
170
            return "%d Mb" % self.get_megabytes(integer_bytes)
171
        elif integer_bytes > 1024*1024*1024 and integer_bytes < (1024*1024*1024*1024):
172
            return "%d Gb" % self.get_gigabytes(integer_bytes)
173
 
174
 
353 t00mlabs 175
    def get_disk_usage_human(self, path):
176
        """
177
        Get disk usage as a fraction for using with Gtk.Progressbar)
178
        Some bytes borrowed from:
179
        https://github.com/juancarlospaco/anglerfish/blob/master/anglerfish/bytes2human.py
356 t00mlabs 180
        """            
353 t00mlabs 181
        total, used, free = self.get_disk_usage(path)
356 t00mlabs 182
        tgb = self.get_gigabytes(total)
183
        ugb = self.get_gigabytes(used)
184
        fgb = self.get_gigabytes(total-used)
353 t00mlabs 185
        humansize = "Using %dGb of %dGb (Free space: %dGb)" % (int(ugb), int(tgb), int(fgb))
186
        return humansize
187
 
340 t00mlabs 188
    def timestamp(self):
189
        """
190
        Missing method docstring (missing-docstring)
191
        """
192
        now = datetime.now()
193
        timestamp = "%4d%02d%02d_%02d%02d%02d" % (now.year, now.month, now.day, now.hour, now.minute, now.second)
194
 
195
        return timestamp
196
 
197
 
198
    def get_datetime(self, timestamp):
199
        """
200
        Missing method docstring (missing-docstring)
201
        """
202
        adate = datetime.strptime(timestamp, "%Y%m%d_%H%M%S")
203
        return adate
204
 
205
 
206
    def get_human_date_from_timestamp(self, timestamp=None):
207
        """
208
        Missing method docstring (missing-docstring)
209
        """
210
        if timestamp is None:
211
            timestamp = self.timestamp()
212
 
213
        adate = self.get_datetime(timestamp)
214
        return "%s" % adate.strftime("%Y/%m/%d %H:%M")
215
 
216
 
217
    def get_excel_date(self, timestamp):
218
        timestamp = timestamp.replace('-', '/')        
219
        timestamp = timestamp.replace('T', ' ')
220
        adate = datetime.strptime(timestamp, "%Y/%m/%d %H:%M:%S")
221
        excel_date = "%s" % adate.strftime("%d/%m/%Y")
222
        return excel_date
223
 
224
 
225
    def fuzzy_date_from_timestamp(self, timestamp):
226
        """
227
        Missing method docstring (missing-docstring)
228
        """
229
        d1 = self.get_datetime(timestamp)
230
        d2 = datetime.now()
231
        rdate = d2 - d1 # DateTimeDelta
232
        if rdate.days > 0:
233
            if rdate.days <= 31:
234
                return "%d days ago" % int(rdate.days)
235
 
236
            if rdate.days > 31 and rdate.days < 365:
237
                return "%d months ago" % int((rdate.days/31))
238
 
239
            if rdate.days >= 365:
240
                return "%d years ago" % int((rdate.days/365))
241
 
242
        hours = rdate.seconds / 3600
243
        if int(hours) > 0:
244
            return "%d hours ago" % int(hours)
245
 
246
        minutes = rdate.seconds / 60
247
        if int(minutes) > 0:
248
            return "%d minutes ago" % int(minutes)
249
 
250
        if int(rdate.seconds) > 0:
251
            return "%d seconds ago" % int(rdate.seconds)
252
 
253
        if int(rdate.seconds) == 0:
254
            return "Right now"
255
 
256
 
257
    def browse(self, url):
258
        """
259
        Missing method docstring (missing-docstring)
260
        """
261
        if sys.platform in ['linux', 'linux2']:
262
            browser = webbrowser.get('firefox')
263
        elif sys.platform == 'win32':
264
            browser = webbrowser.get('windows-default')
265
 
266
        browser.open_new_tab(url)
267
 
268
 
269
    def which(self, program):
270
        """
271
        Missing method docstring (missing-docstring)
272
        """
273
        if sys.platform == 'win32':
274
            program = program + '.exe'
275
 
276
        def is_exe(fpath):
277
            """
278
            Missing method docstring (missing-docstring)
279
            """
280
            return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
281
 
282
        fpath, fname = os.path.split(program)
283
        if fpath:
284
            if is_exe(program):
285
                return program
286
        else:
287
            for path in os.environ["PATH"].split(os.pathsep):
288
                path = path.strip('"')
289
                exe_file = os.path.join(path, program)
290
                if is_exe(exe_file):
291
                    return exe_file
292
 
293
        return None
294
 
295
 
296
    def install_geckodriver(self):
297
        """
298
        Install Gecko Driver
299
        """
300
        GECKODRIVER_LINUX = GPATH['DRIVERS'] + 'geckodriver'
301
        GECKODRIVER_WIN32 = GPATH['DRIVERS'] + 'geckodriver.exe'
302
        GECKO_INSTALL_DIR = LPATH['DRIVERS']
303
 
304
        if sys.platform == 'linux':
305
            shutil.copy(GECKODRIVER_LINUX, GECKO_INSTALL_DIR)
306
        else:
307
            shutil.copy(GECKODRIVER_WIN32, GECKO_INSTALL_DIR)
308
        self.log.debug("Gecko driver installed to %s" % GECKO_INSTALL_DIR)
309
 
310
 
311
    def download(self, prgname, source, target):
312
        """
313
        Missing method docstring (missing-docstring)
314
        """
315
        try:
316
            self.log.debug("Downloading %s from: %s" % (prgname, source))
317
            response = requests.get(source, stream=True)
318
            with open(target, 'wb') as out_file:
319
                shutil.copyfileobj(response.raw, out_file)
320
            del response
321
            self.log.debug("%s downloaded to %s" % (prgname, target))
322
            return True
323
        except Exception as error:
324
            self.log.error(error)
325
            return False
326
 
327
 
328
    def extract(self, filename, target_path, protocol):
329
        """
330
        Missing method docstring (missing-docstring)
331
        """
332
        self.log.debug("Extracting %s to %s using protocol %s" % (filename, target_path, protocol))
333
        if protocol in ['tar.gz', 'bz2']:
334
            try:
335
                tar = tarfile.open(filename, "r:*")
336
                tar.extractall(target_path)
337
                tar.close()
338
                self.log.debug("Extracted successfully")
339
                return True
340
            except Exception as error:
341
                self.log.error(error)
342
                return False
343
        elif protocol == 'zip':
344
            try:
345
                self.unzip(filename, target_path)
346
                self.log.debug("Extracted successfully")
347
                return True
348
            except Exception as error:
349
                self.log.error(error)
350
                return False
351
 
352
    def zip(self, filename, directory):
353
        """
354
        Zip directory and rename to .bco
355
        """
356
        # http://stackoverflow.com/a/25650295
357
        #~ make_archive(archive_name, 'gztar', root_dir)
358
        self.log.debug("Target: %s", filename)
359
        sourcename = os.path.basename(filename)
360
        dot = sourcename.find('.')
361
        if dot == -1:
362
            basename = sourcename
363
        else:
364
            basename = sourcename[:dot]
365
        sourcedir = os.path.dirname(filename)
366
        source = os.path.join(sourcedir, basename)
367
        zipfile = shutil.make_archive(source, 'zip', directory)
368
        target = source + '.bco'
369
        shutil.move(zipfile, target)
370
        return target
371
 
372
 
373
    def unzip(self, target, install_dir):
374
        """
375
        Unzip file to a given dir
376
        """
377
        zip_archive = zipfile.ZipFile(target, "r")
378
        zip_archive.extractall(path=install_dir)
379
        zip_archive.close()
380
 
381
 
382
    def get_firefox_profile_dir(self):
383
        """
384
        Self-explained. Get default Firefox directory
385
        """
386
        if sys.platform in ['linux', 'linux2']:
387
            cmd = "ls -d /home/$USER/.mozilla/firefox/*.default/"
388
            p = subprocess.Popen([cmd], shell=True, stdout=subprocess.PIPE)
389
            FF_PRF_DIR = p.communicate()[0][0:-2]
390
            FF_PRF_DIR_DEFAULT = str(FF_PRF_DIR, 'utf-8')
391
        elif sys.platform == 'win32':
392
            import glob
393
            APPDATA = os.getenv('APPDATA')
394
            FF_PRF_DIR = "%s\\Mozilla\\Firefox\\Profiles\\" % APPDATA
395
            PATTERN = FF_PRF_DIR + "*default*"
396
            FF_PRF_DIR_DEFAULT = glob.glob(PATTERN)[0]
397
 
398
        return FF_PRF_DIR_DEFAULT
399
 
400
 
401
    def feedparser_parse(self, thing):
402
        """
403
        Missing method docstring (missing-docstring)
404
        """
405
        try:
406
            return feedparser.parse(thing)
407
        except TypeError:
408
            if 'drv_libxml2' in feedparser.PREFERRED_XML_PARSERS:
409
                feedparser.PREFERRED_XML_PARSERS.remove('drv_libxml2')
410
                return feedparser.parse(thing)
411
            else:
412
                self.log.error(self.get_traceback())
413
                return None
414
 
415
 
416
    def check(self):
417
        """
418
        Check Basico environment
419
        """
420
        gtk_version = self.uif.check_gtk_version() # Check GTK version
421
        gecko_driver = self.driver.check() # Check Gecko webdrver
422
        run = gtk_version and gecko_driver
423
        if run:
424
            self.log.debug("Basico environment ready!")
425
        else:
426
            self.log.error("Error(s) found checking Basico environment")
427
 
428
        return run
429
 
430
 
431
 
432
 
433
    def clean_html(self, raw_html):
434
        cleanr = re.compile('<.*?>')
435
        cleantext = re.sub(cleanr, '', raw_html)
436
        return cleantext