Subversion Repositories basico

Rev

Rev 377 | Details | Compare with Previous | Last modification | View Log | RSS feed

Rev Author Line No. Line
340 t00mlabs 1
#!/usr/bin/python
2
# -*- coding: utf-8 -*-
3
"""
4
# File: srv_db.py
5
# Author: Tomás Vírseda
6
# License: GPL v3
7
# Description: Database service
8
"""
9
 
10
import os
11
from os.path import basename
12
import json
13
import glob
14
from html import escape
15
 
16
from basico.core.mod_env import LPATH, FILE
17
from basico.core.mod_srv import Service
363 t00mlabs 18
from basico.services.srv_collections import COL_DOWNLOADED
340 t00mlabs 19
 
20
SEP = os.path.sep
21
 
22
class Database(Service):
23
    def initialize(self):
24
        '''
25
        Setup AppLogic Service
26
        '''
27
        self.sapnotes = {}
28
        self.stats = {}
29
        self.stats['maincomp'] = {}
30
        self.stats['cats'] = {}
31
        self.stats['component'] = set()
32
        self.stats['category'] = set()
33
        self.stats['priority'] = set()
34
        self.stats['type'] = set()
35
        self.stats['version'] = set()
36
        self.__init_config_section()
37
        self.get_services()
38
        self.load_notes()
39
 
40
 
41
    def __init_config_section(self):
42
        settings = self.srvstg.load()
43
        settings[self.section]
44
        try:
45
            settings[self.section]['DBSAP']
46
        except:
47
            settings[self.section]['DBSAP'] = FILE['DBSAP']
48
        self.srvstg.save(settings)
49
 
374 t00mlabs 50
 
340 t00mlabs 51
    def get_services(self):
52
        self.srvclt = self.get_service('Collections')
53
 
54
    def store(self, sapnote, html):
55
        FSAPNOTE = LPATH['CACHE_XML'] + sapnote + '.xml'
56
 
57
        try:
58
            f = open(FSAPNOTE, 'w')
59
            f.write(html)
60
            f.close()
61
            self.log.debug("SAP Note %s in XML format stored in %s" % (sapnote, FSAPNOTE))
62
        except Exception as error:
63
            self.log.error(error)
64
 
65
 
66
    def get_sapnote_content(self, sid):
67
        FSAPNOTE = LPATH['CACHE_XML'] + sid + '.xml'
68
        content = open(FSAPNOTE, 'r').read()
69
        return content
70
 
71
 
72
    def is_saved(self, sid):
73
        metadata = self.get_sapnote_metadata(sid)
74
        if metadata is None:
75
            return False
76
        else:
77
            return True
78
 
79
 
80
    def is_valid(self, sid):
81
        valid = True
82
        for ch in sid:
83
            if not ch.isdigit():
84
                return False
85
        return valid
86
 
87
 
88
    def is_stored(self, sid):
89
        fsapnote = LPATH['CACHE_XML'] + sid + '.xml'
90
        stored = os.path.exists(fsapnote)
91
 
92
        return stored
93
 
94
 
95
    def build_stats(self, bag=None):
96
        if bag is None:
97
            bag = self.sapnotes
98
        self.dstats = {}
99
        self.compstats = {}
100
        allcollections = set()
101
 
102
        for sid in bag:
103
            try:
104
                collections = self.sapnotes[sid]['collections']
105
                for collection in collections:
106
                    allcollections.add(collection)
107
            except: pass
108
 
109
            #COMMENT Components
110
            compkey = self.sapnotes[sid]['componentkey']
111
            comptxt = self.sapnotes[sid]['componenttxt']
112
            component = escape("%s (%s)" % (compkey, comptxt))
113
            sep = compkey.find('-')
114
            if sep > 0:
115
                maincomp = compkey[0:sep]
116
            else:
117
                maincomp = compkey
118
 
119
            #COMMENT Categories
120
            category = escape(self.sapnotes[sid]['category'])
121
            try:
122
                cont = self.stats['cats'][category]
123
                self.stats['cats'][category] = cont + 1
124
            except:
125
                self.stats['cats'][category] = 1
126
 
127
            #COMMENT Build all (sub)keys from given component key
128
            #COMMENT useful for display stats with pygal
129
            compkeys = compkey.split('-')
130
            total = len(compkeys)
131
            key = ''
132
            i = 0
133
            for subkey in compkeys:
134
                key = key + '-' + subkey
135
                if key[:1] == '-':
136
                    key = key[1:]
137
 
138
                # update stats
139
                try:
140
                    count = self.compstats[key]
141
                    self.compstats[key] = count + 1
142
                except Exception as error:
143
                    self.compstats[key] = 1
144
 
145
            try:
146
                cont = self.stats['maincomp'][maincomp]
147
                self.stats['maincomp'][maincomp] = cont + 1
148
            except:
149
                self.stats['maincomp'][maincomp] = 1
150
 
151
            category = escape(self.sapnotes[sid]['category'])
152
            priority = escape(self.sapnotes[sid]['priority'])
153
            ntype = escape(self.sapnotes[sid]['type'])
154
            version = escape(self.sapnotes[sid]['version'])
155
            releaseon = escape(self.sapnotes[sid]['releasedon'])
156
            self.stats['component'].add(component)
157
            self.stats['category'].add(category)
158
            self.stats['priority'].add(priority)
159
            self.stats['type'].add(ntype)
160
            self.stats['version'].add(version)
161
            #FIXME self.stats['releaseon'].add(releasedon)
162
            #FIXME self.stats[''].add(version)
163
 
164
 
165
    def add(self, sapnote, overwrite=False, batch=False):
166
        sid = sapnote['id']
167
 
168
        if self.exists(sid):
169
            if overwrite:
170
                self.sapnotes[sid] = sapnote
171
                self.log.info("SAP Note %s added to database (Overwrite is %s)", sid, overwrite)
172
                if batch is False:
173
                    self.save_notes()
174
                return True
175
            else:
176
                self.log.info("SAP Note %s already exists in database (Overwrite is %s)", sid, overwrite)
177
                return False
178
        else:
179
            self.sapnotes[sid] = sapnote
180
            self.log.info("SAP Note %s added to database (Overwrite is %s)", sid, overwrite)
181
            if batch is False:
182
                self.save_notes()
183
            return True
184
 
185
 
186
    def add_list(self, sapnotes, overwrite=False):
187
        self.log.info("Add %d SAP Notes to database (Overwrite is %s)", len(sapnotes), overwrite)
188
        n = 0
189
        for sid in sapnotes:
190
            if overwrite:
191
                if self.add(sapnotes[sid], overwrite, batch=True):
192
                    n += 1
193
        self.save_notes()
194
        return n
195
 
374 t00mlabs 196
 
340 t00mlabs 197
    def get_notes(self):
198
        '''
199
        Return all sapnotes
200
        '''
201
        return self.sapnotes
202
 
203
 
204
    def get_notes_by_node(self, key, value):
205
        '''
206
        Return a set of sapnotes which matches with an specific key/value
207
        '''
208
        bag = set()
209
        sapnotes = self.get_notes()
210
        for sapnote in sapnotes:
211
            if key.startswith('component') or \
212
               key in ['category', 'priority', 'type']:
213
                if sapnotes[sapnote][key].startswith(value):
214
                    bag.add(sapnote)
215
            elif key == 'collection':
216
                try:
217
                    collections = sapnotes[sapnote]['collections']
218
                    if value == 'None':
219
                        if len(collections) == 0:
220
                            bag.add(sapnote)
221
                    else:
222
                        if value in collections:
223
                            bag.add(sapnote)
224
                except:
225
                    pass
226
            elif key in ['date-year', 'date-month', 'date-day']:
227
                if len(value) == 6:
228
                    value = value[0:4] + '-' + value[4:6]
229
                elif len(value) == 8:
230
                    value = value[0:4] + '-' + value[4:6] + '-' + value[6:8]
231
 
232
                if sapnotes[sapnote]['feedupdate'].startswith(value):
233
                    bag.add(sapnote)
234
            else:
235
                pass
236
 
237
        return bag
238
 
239
 
240
    def get_total(self):
241
        '''
242
        Return total sapnotes
243
        '''
244
        return len(self.sapnotes)
245
 
246
 
247
    def load_notes(self):
248
        '''
249
        If notes.json exists, load notes
250
        '''
251
        try:
252
            with open(FILE['DBSAP'], 'r') as fp:
253
                self.sapnotes = json.load(fp)
254
                self.log.info("SAP Notes Database found at: %s", FILE['DBSAP'])
255
                self.log.info("Loaded %d notes from SAP Notes database" % len(self.sapnotes))
256
        except Exception as error:
257
            self.save_notes()
258
            self.log.info("SAP Notes database not found. Created a new database for SAP Notes")
259
 
260
    def get_sapnote_metadata(self, sid):
261
        sid = self.normalize_sid(sid)
262
 
263
        try:
264
            return self.sapnotes[sid]
265
        except KeyError as error:
266
            # ~ self.log.debug("SAP Note %s doesn't exist in the database or it's been deleted" % sid)
267
            return None
268
 
269
 
270
    def exists(self, sid):
271
        sid = self.normalize_sid(sid)
272
        metadata = self.get_sapnote_metadata(sid)
273
        if metadata is not None:
274
            return True
275
        else:
276
            return False
277
 
278
    def get_title(self, sid):
279
        title = ''
280
        metadata = self.get_sapnote_metadata(sid)
281
        if metadata is not None:
282
            title = metadata['title']
283
 
284
        return title
285
 
286
 
287
    def get_component(self, sid):
288
        component = ''
289
        metadata = self.get_sapnote_metadata(sid)
290
        if metadata is not None:
383 t00mlabs 291
            component = metadata['componentkey']
340 t00mlabs 292
 
293
        return component
294
 
295
 
296
    def save_notes(self):
297
        '''
298
        Save SAP Notes to json database file
299
        '''
300
 
301
        bag = self.get_notes()
302
 
303
        try:
304
            with open(FILE['DBSAP'], 'w') as fdb:
305
                json.dump(bag, fdb)
306
                self.log.debug ("Saved %d notes to %s" % (len(bag), FILE['DBSAP']))
307
        except Exception as error:
308
            self.log.error(error)
309
 
310
 
311
    def set_bookmark(self, lsid):
312
        for sid in lsid:
313
            sid = self.normalize_sid(sid)
314
            self.sapnotes[sid]['bookmark'] = True
315
            self.log.info("SAP Note %s bookmarked: True" % sid)
316
        self.save_notes()
317
 
318
 
319
    def set_no_bookmark(self, lsid):
320
        for sid in lsid:
321
            sid = self.normalize_sid(sid)
322
            self.sapnotes[sid]['bookmark'] = False
323
            self.log.info("SAP Note %s bookmarked: False" % sid)
324
        self.save_notes()
325
 
326
 
327
    def is_bookmark(self, sapnote):
328
        try:
329
            return self.sapnotes[sapnote]['bookmark']
330
        except:
331
            return False
332
 
333
    def get_collections(self, sid):
334
        try:
335
            return self.sapnotes[sid]['collections']
336
        except:
337
            return []
338
 
339
 
340
    def get_stats(self):
341
        return self.stats
342
 
343
 
344
    def search(self, term):
345
        bag = []
346
 
347
        for sid in self.get_notes():
348
            values = []
349
            sapnote = self.sapnotes[sid]
350
            for key in sapnote:
351
                values.append(str(sapnote[key]))
352
            text = ' '.join(values)
353
            if term.upper() in text.upper():
354
                bag.append(sapnote['id'])
355
        self.log.debug("Found term '%s' in %d SAP Notes" % (term, len(bag)))
356
        return bag
357
 
358
 
359
    def normalize_sid(self, sid):
360
        if isinstance(sid, int):
361
            sid = "0"*(10 - len(str(sid))) + str(sid)
362
        elif isinstance(sid, str):
363
            sid = "0"*(10 - len(sid)) + sid
364
 
365
        return sid
366
 
367
 
368
    def set_collections(self, sid, collections, overwrite=False):
369
        sid = self.normalize_sid(sid)
370
        self.log.info("SAP Note %s added to the following collections (overwrite mode is %s):", sid, overwrite)
371
        if sid != '0000000000':
372
            sid = self.normalize_sid(sid)
373
            try:
374
                if overwrite:
375
                    new_cols = []
376
                    self.sapnotes[sid]['collections'] = collections
377
                    for cid in collections:
378
                        if cid == COL_DOWNLOADED:
379
                            continue
380
                        if cid not in new_cols:
381
                            new_cols.append(cid)
374 t00mlabs 382
                            self.log.info("\tCollection: %s", self.srvclt.get_name_by_cid(cid))
340 t00mlabs 383
                else:
384
                    current_collections = self.sapnotes[sid]['collections']
385
                    current_collections.extend(collections)
386
                    bag = []
387
                    for cid in current_collections:
388
                        if cid == COL_DOWNLOADED:
389
                            continue
390
                        if cid not in bag:
391
                            bag.append(cid)
374 t00mlabs 392
                            self.log.info("\tCollection: %s", self.srvclt.get_name_by_cid(cid))
340 t00mlabs 393
                    self.sapnotes[sid]['collections'] = bag
394
                self.save_notes()
395
            except:
396
                self.log.error(error)
397
 
374 t00mlabs 398
 
340 t00mlabs 399
    def delete(self, sid):
400
        sid = self.normalize_sid(sid)
401
        deleted = False
402
        try:
403
            del (self.sapnotes[sid])
404
            deleted = True
405
            self.log.info("SAP Note %s deleted" % sid)
406
            self.save_notes()
407
        except Exception as error:
408
            self.log.debug(error)
409
            deleted = False
410
 
411
        return deleted
412
 
413
 
414
    def end(self):
415
        self.save_notes()
416