Subversion Repositories basico

Rev

Rev 73 | Details | Compare with Previous | Last modification | View Log | RSS feed

Rev Author Line No. Line
34 t00mlabs 1
#!/usr/bin/python
2
# -*- coding: utf-8 -*-
3
# File: database.py
4
# Author: Tomás Vírseda
5
# License: GPL v3
6
# Description: Database service
7
 
8
import os
42 t00mlabs 9
from os.path import basename
34 t00mlabs 10
#~ import sys
11
#~ import time
12
import json
42 t00mlabs 13
import glob
65 t00mlabs 14
#~ import shutil
34 t00mlabs 15
#~ import traceback
16
#~ from shutil import which
17
from cgi import escape
18
 
57 t00mlabs 19
from .env import LPATH
34 t00mlabs 20
from .service import Service
21
 
57 t00mlabs 22
SEP = os.path.sep
34 t00mlabs 23
 
24
class Database(Service):
25
    def initialize(self):
26
        '''
27
        Setup AppLogic Service
28
        '''
29
        self.sapnotes = {}
30
        self.stats = {}
31
        self.stats['maincomp'] = {}
32
        self.stats['cats'] = {}
33
        self.stats['component'] = set()
34
        self.stats['category'] = set()
35
        self.stats['priority'] = set()
36
        self.stats['type'] = set()
37
        self.stats['version'] = set()
38
        #~ self.stats['releaseon'] = set()
42 t00mlabs 39
        #~ self.rebuild_from_dir()
34 t00mlabs 40
 
41
 
42
    def store(self, sapnote, html):
43
        DB_DIR = self.get_var('DB', 'local')
44
        FSAPNOTE = DB_DIR + sapnote + '.xml'
45
 
73 t00mlabs 46
        try:
47
            f = open(FSAPNOTE, 'w')
48
            f.write(html)
49
            f.close()
50
            self.log.debug("\tSAP Note %s stored in %s" % (sapnote, FSAPNOTE))
51
        except Exception as error:
52
            self.log.error(error)
34 t00mlabs 53
 
73 t00mlabs 54
 
42 t00mlabs 55
    def rebuild_from_dir(self, path=None):
56
        db = self.get_service('DB')
57
        DB_DIR = self.get_var('DB', 'local')
58
 
59
        #~ FSAPNOTE = DB_DIR + sid + '.xml'
60
        if path is None:
61
            path = DB_DIR
62
 
63
        files = glob.glob("%s%s*.xml" % (DB_DIR, SEP))
64
        sap = self.app.get_service('SAP')
65
        for filename in files:
66
            self.log.debug("Filename: %s" % filename)
67
            sid = basename(filename)[0:-4]
68
            self.log.debug("SAP Note Id: %s" % sid)
69
 
70
            valid = False
71
            if db.is_stored(sid):
72
                self.log.debug("\tSAP Note %s will be analyzed again" % sid)
73
                content = db.get_sapnote_content(sid)
74
                sapnote = sap.analyze_sapnote_metadata(sid, content)
75
                if len(sapnote) > 0:
76
                    db = self.get_service('DB')
77
                    db.add(sapnote)
78
                    db.store(sid, content)
79
                    valid = True
80
 
81
        cb =self.app.get_service('Callbacks')
82
        cb.refresh_view()
83
 
84
 
34 t00mlabs 85
    def get_sapnote_content(self, sid):
86
        DB_DIR = self.get_var('DB', 'local')
87
        FSAPNOTE = DB_DIR + sid + '.xml'
88
        content = open(FSAPNOTE, 'r').read()
89
        return content
90
 
91
 
92
    def is_stored(self, sid):
93
        DB_DIR = self.get_var('DB', 'local')
94
        fsapnote = DB_DIR + sid + '.xml'
95
        stored = os.path.exists(fsapnote)
73 t00mlabs 96
        #~ self.log.debug("\tSAP Note %s stored in database? %s" % (sid, stored))
34 t00mlabs 97
 
98
        return stored
99
 
100
 
101
    def build_stats(self, bag=None):
102
        if bag is None:
103
            bag = self.sapnotes
104
        self.dstats = {}
105
        self.compstats = {}
106
        alltasks = set()
107
 
108
        for sid in bag:
109
            # tasks
110
            try:
111
                tasks = self.sapnotes[sid]['tasks']
112
                for task in tasks:
113
                    alltasks.add(task)
114
            except: pass
115
            self.tasks = self.app.get_service('Tasks')
116
            self.tasks.save_tasks_from_stats(alltasks)
117
 
118
            # components
119
            compkey = self.sapnotes[sid]['componentkey']
120
            comptxt = self.sapnotes[sid]['componenttxt']
73 t00mlabs 121
            #~ self.log.debug("%s -> %s -> %s" % (sid, compkey, comptxt))
34 t00mlabs 122
            component = escape("%s (%s)" % (compkey, comptxt))
123
            sep = compkey.find('-')
124
            if sep > 0:
125
                maincomp = compkey[0:sep]
126
            else:
127
                maincomp = compkey
128
 
129
            # categories
41 t00mlabs 130
            category = escape(self.sapnotes[sid]['category'])
34 t00mlabs 131
            try:
132
                cont = self.stats['cats'][category]
133
                self.stats['cats'][category] = cont + 1
134
            except:
135
                self.stats['cats'][category] = 1
136
 
41 t00mlabs 137
            # Build all (sub)keys from given component key
34 t00mlabs 138
            # useful for display stats with pygal
139
            compkeys = compkey.split('-')
140
            total = len(compkeys)
141
            key = ''
142
            i = 0
143
            for subkey in compkeys:
144
                key = key + '-' + subkey
145
                if key[:1] == '-':
146
                    key = key[1:]
147
 
148
                # update stats
149
                try:
150
                    count = self.compstats[key]
151
                    self.compstats[key] = count + 1
152
                except Exception as error:
153
                    self.compstats[key] = 1
154
 
155
            try:
156
                cont = self.stats['maincomp'][maincomp]
157
                self.stats['maincomp'][maincomp] = cont + 1
158
            except:
159
                self.stats['maincomp'][maincomp] = 1
160
 
161
            category = escape(self.sapnotes[sid]['category'])
162
            priority = escape(self.sapnotes[sid]['priority'])
163
            ntype = escape(self.sapnotes[sid]['type'])
164
            version = escape(self.sapnotes[sid]['version'])
165
            releaseon = escape(self.sapnotes[sid]['releaseon'])
166
            self.stats['component'].add(component)
167
            self.stats['category'].add(category)
168
            self.stats['priority'].add(priority)
169
            self.stats['type'].add(ntype)
170
            self.stats['version'].add(version)
171
            #~ self.stats['releaseon'].add(releaseon)
172
            #~ self.stats[''].add(version)
173
        #~ self.log.debug(self.compstats)
174
        #~ self.log.debug("==")
175
        #~ self.log.debug(self.stats)
41 t00mlabs 176
        #~ self.log.debug(self.stats['maincomp'])
34 t00mlabs 177
 
178
 
179
    def get_stats(self):
180
        return self.stats
181
 
182
 
183
    def add(self, sapnote):
184
        sid = sapnote['id']
185
        self.sapnotes[sid] = sapnote
186
 
187
 
39 t00mlabs 188
    def add_list(self, sapnotes):
189
        for sapnote in sapnotes:
190
            sid = sapnote['id']
191
            self.sapnotes[sid] = sapnote
192
 
193
 
34 t00mlabs 194
    def get_notes(self):
195
        '''
196
        Return all sapnotes
197
        '''
198
        return self.sapnotes
199
 
200
 
201
    def get_total(self):
202
        '''
203
        Return total sapnotes
204
        '''
205
        return len(self.sapnotes)
206
 
207
 
208
    def load_notes(self):
209
        '''
210
        If notes.json exists, load notes
211
        '''
212
        try:
213
            fnotes = self.get_file('SAP')
214
            with open(fnotes, 'r') as fp:
215
                self.sapnotes = json.load(fp)
216
                self.log.debug ("Loaded %d notes" % len(self.sapnotes))
217
        except Exception as error:
218
            self.log.info("SAP Notes database not found. Creating a new one")
219
            self.save_notes()
220
 
221
 
222
    def get_sapnote_metadata(self, sid):
223
        try:
224
            return self.sapnotes[sid]
41 t00mlabs 225
        except KeyError as error:
61 t00mlabs 226
            self.log.warning("SAP Note %s doesn't exist in the database" % sid)
34 t00mlabs 227
            return None
228
 
57 t00mlabs 229
    #~ def save_notes(self, filename='', bag={}):
230
        #~ '''
231
        #~ Save SAP Notes to file
232
        #~ '''
233
        #~ if len(filename) == 0:
234
            #~ filename = self.get_file('SAP')
235
            #~ bag = self.sapnotes
34 t00mlabs 236
 
57 t00mlabs 237
        #~ fnotes = open(filename, 'w')
238
        #~ json.dump(bag, fnotes)
239
        #~ fnotes.close()
240
        #~ self.log.info ("Saved %d notes to %s" % (len(bag), filename))
34 t00mlabs 241
 
242
 
39 t00mlabs 243
    def save_notes(self, bag={}, export_path=None):
244
        '''
245
        Save SAP Notes to json database file
246
        '''
247
        if export_path is None:
248
            export_path = self.get_file('SAP')
249
 
250
        if len(bag) == 0:
251
            bag = self.get_notes()
252
 
253
        fdb = open(export_path, 'w')
254
        json.dump(bag, fdb)
255
        fdb.close()
256
        self.log.info ("Saved %d notes to %s" % (len(bag), export_path))
257
 
258
 
259
    def export(self, bag, export_path=None):
260
        pass
261
 
57 t00mlabs 262
 
58 t00mlabs 263
    def export_basico_package(self, bag, target):
264
        self.save_notes(bag, target)
265
        #~ TMP_DIR = LPATH['TMP']
266
        #~ PKG_DIR = TMP_DIR + 'basico'
267
        #~ BCO_SAPNOTES = PKG_DIR + SEP + 'sapnotes.json'
268
        #~ TARGET = export_path + SEP + '.bco'
57 t00mlabs 269
 
270
        # 0 Delete temporary package dir
58 t00mlabs 271
        #~ try:
272
            #~ shutil.rmtree(PKG_DIR)
273
            #~ self.log.debug("Temporary package dir deleted: '%s'" % PKG_DIR)
274
        #~ except: pass
57 t00mlabs 275
 
276
        # 1 Create temporary dir for building a basico package
58 t00mlabs 277
        #~ os.mkdir(PKG_DIR)
278
        #~ self.log.debug("Temporary package dir created: '%s'" % PKG_DIR)
57 t00mlabs 279
 
280
        # 2 Export selected SAP Notes to basico package directory
58 t00mlabs 281
        #~ self.save_notes(bag, BCO_SAPNOTES)
57 t00mlabs 282
 
283
        # 3 Create Basico Package
58 t00mlabs 284
        #~ utils = self.get_service('Utils')
285
        #~ utils.zip('basico_package', PKG_DIR)
57 t00mlabs 286
 
287
        # 4 Delete temporary dir
288
        #~ shutil.rmtree(PKG_DIR)
289
        #~ self.log.debug("Temporary package dir deleted: '%s'" % PKG_DIR)
290
 
291
 
292
    def get_property_count(self, prop):
293
        found = 0
294
 
295
        store = self.get_notes()
296
        for sid in store:
297
            try:
298
                store[sid][prop]
299
                found += 1
300
            except:
301
                pass
302
 
303
        return found
304
 
305
 
34 t00mlabs 306
    def get_linked_projects(self, sapnote):
307
        try:
308
            projects = self.sapnotes[sapnote]['projects']
309
        except Exception as error:
310
            projects = []
311
 
312
        return projects
313
 
314
 
315
    def get_linked_tasks(self, sapnote):
316
        try:
317
            tasks = self.sapnotes[sapnote]['tasks']
318
        except Exception as error:
319
            tasks = []
320
        self.log.debug("Tasks: %s" % tasks)
321
        return tasks
322
 
323
 
324
    def link_to_task(self, sapnotes, tasks):
325
        for sapnote in sapnotes:
326
            try:
327
                self.sapnotes[sapnote]['tasks'] = tasks
328
                self.log.info("Linked SAP Note %s to task(s): %s" % (sapnote, tasks) )
329
            except:
330
                self.log.error(self.get_traceback())
331
 
332
 
333
    def import_sapnotes(self, bag):
334
        for sid in bag:
335
            # Check if SAP Note exists in main database
61 t00mlabs 336
            found = self.get_sapnote_metadata(sid)
34 t00mlabs 337
            if found is None:
338
                self.sapnotes[sid] = bag[sid]
339
            else:
340
                # Import only tasks
341
                try:
342
                    imptasks = bag[sid]['tasks']
343
                    tasks = self.sapnotes[sid]['tasks']
344
                    tasks.extend(imptasks)
345
                    self.sapnotes[sid]['tasks'] = tasks
61 t00mlabs 346
                except Exception as error:
347
                    self.log.error(error)
348
                    pass
34 t00mlabs 349
                # Import other metadata
350
 
351
 
352
    def set_bookmark(self, sapnotes):
353
        for sapnote in sapnotes:
354
            self.log.info("SAP Note %s bookmarked: True" % sapnote)
355
            self.sapnotes[sapnote]['bookmark'] = True
356
        self.save_notes()
357
 
358
 
359
    def set_no_bookmark(self, sapnotes):
360
        for sapnote in sapnotes:
361
            self.log.info("SAP Note %s bookmarked: False" % sapnote)
362
            self.sapnotes[sapnote]['bookmark'] = False
363
        self.save_notes()
364
 
365
 
366
    def is_bookmark(self, sapnote):
367
        try:
368
            return self.sapnotes[sapnote]['bookmark']
369
        except:
370
            return False
371
 
372
 
373
    def delete(self, sapnote):
374
        deleted = False
375
        try:
376
            del (self.sapnotes[sapnote])
377
            deleted = True
378
        except:
379
            deleted = False
380
 
381
        return deleted
382
 
383
 
384
    def run(self):
385
        self.load_notes()
386
        self.build_stats()
387
 
388
 
389
    def end(self):
390
        self.save_notes()
391