[IMP] ir.attachment: filestore is now in data-dir and by default
bzr revid: chs@openerp.com-20140116185415-ajia02bsty9joox7
This commit is contained in:
Christophe Simonis
parent
c48f11041b
commit
3275de5981
|
|
@ -63,19 +63,38 @@ class ir_attachment(osv.osv):
|
|||
data[attachment.id] = False
|
||||
return data
|
||||
|
||||
def _storage(self, cr, uid, context=None):
|
||||
return self.pool['ir.config_parameter'].get_param(cr, SUPERUSER_ID, 'ir_attachment.location', 'file')
|
||||
|
||||
@tools.ormcache()
|
||||
def _filestore(self, cr, uid, context=None):
|
||||
dbuuid = self.pool['ir.config_paramater'].get_param(cr, SUPERUSER_ID, 'database.uuid')
|
||||
return os.path.join(tools.config['data_dir'], 'filestore', dbuuid)
|
||||
|
||||
# 'data' field implementation
|
||||
def _full_path(self, cr, uid, location, path):
|
||||
# location = 'file:filestore'
|
||||
assert location.startswith('file:'), "Unhandled filestore location %s" % location
|
||||
location = location[5:]
|
||||
|
||||
# sanitize location name and path
|
||||
location = re.sub('[.]','',location)
|
||||
location = location.strip('/\\')
|
||||
|
||||
path = re.sub('[.]','',path)
|
||||
# sanitize ath
|
||||
path = re.sub('[.]', '', path)
|
||||
path = path.strip('/\\')
|
||||
return os.path.join(tools.config['root_path'], location, cr.dbname, path)
|
||||
return os.path.join(self._filestore(cr, uid), path)
|
||||
|
||||
def _get_path(self, cr, uid, location, bin_data):
|
||||
sha = hashlib.sha1(bin_data).hexdigest()
|
||||
|
||||
# retro compatibility
|
||||
fname = sha[:3] + '/' + sha
|
||||
full_path = self._full_path(cr, uid, location, fname)
|
||||
if os.path.isfile(full_path):
|
||||
return fname, full_path # keep existing path
|
||||
|
||||
# scatter files across 256 dirs
|
||||
# we use '/' in the db (even on windows)
|
||||
fname = sha[:2] + '/' + sha
|
||||
full_path = self._full_path(cr, uid, location, fname)
|
||||
dirname = os.path.dirname(full_path)
|
||||
if not os.path.isdir(dirname):
|
||||
os.makedirs(dirname)
|
||||
return fname, full_path
|
||||
|
||||
def _file_read(self, cr, uid, location, fname, bin_size=False):
|
||||
full_path = self._full_path(cr, uid, location, fname)
|
||||
|
|
@ -91,18 +110,13 @@ class ir_attachment(osv.osv):
|
|||
|
||||
def _file_write(self, cr, uid, location, value):
|
||||
bin_value = value.decode('base64')
|
||||
fname = hashlib.sha1(bin_value).hexdigest()
|
||||
# scatter files across 1024 dirs
|
||||
# we use '/' in the db (even on windows)
|
||||
fname = fname[:3] + '/' + fname
|
||||
full_path = self._full_path(cr, uid, location, fname)
|
||||
try:
|
||||
dirname = os.path.dirname(full_path)
|
||||
if not os.path.isdir(dirname):
|
||||
os.makedirs(dirname)
|
||||
open(full_path,'wb').write(bin_value)
|
||||
except IOError:
|
||||
_logger.error("_file_write writing %s",full_path)
|
||||
fname, full_path = self._get_path(cr, uid, location, bin_value)
|
||||
if not os.path.exists(full_path):
|
||||
try:
|
||||
with open(full_path, 'wb') as fp:
|
||||
fp.write(bin_value)
|
||||
except IOError:
|
||||
_logger.error("_file_write writing %s", full_path)
|
||||
return fname
|
||||
|
||||
def _file_delete(self, cr, uid, location, fname):
|
||||
|
|
@ -121,7 +135,7 @@ class ir_attachment(osv.osv):
|
|||
if context is None:
|
||||
context = {}
|
||||
result = {}
|
||||
location = self.pool.get('ir.config_parameter').get_param(cr, uid, 'ir_attachment.location')
|
||||
location = self._storage(cr, uid, context)
|
||||
bin_size = context.get('bin_size')
|
||||
for attach in self.browse(cr, uid, ids, context=context):
|
||||
if location and attach.store_fname:
|
||||
|
|
@ -136,7 +150,7 @@ class ir_attachment(osv.osv):
|
|||
return True
|
||||
if context is None:
|
||||
context = {}
|
||||
location = self.pool.get('ir.config_parameter').get_param(cr, uid, 'ir_attachment.location')
|
||||
location = self._storage(cr, uid, context)
|
||||
file_size = len(value.decode('base64'))
|
||||
if location:
|
||||
attach = self.browse(cr, uid, id, context=context)
|
||||
|
|
@ -284,7 +298,7 @@ class ir_attachment(osv.osv):
|
|||
if isinstance(ids, (int, long)):
|
||||
ids = [ids]
|
||||
self.check(cr, uid, ids, 'unlink', context=context)
|
||||
location = self.pool.get('ir.config_parameter').get_param(cr, uid, 'ir_attachment.location')
|
||||
location = self._storage(cr, uid, context)
|
||||
if location:
|
||||
for attach in self.browse(cr, uid, ids, context=context):
|
||||
if attach.store_fname:
|
||||
|
|
@ -302,4 +316,3 @@ class ir_attachment(osv.osv):
|
|||
cr, uid, 'base', 'action_attachment', context=context)
|
||||
|
||||
# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
|
||||
|
||||
|
|
|
|||
|
|
@ -1,89 +1,91 @@
|
|||
import hashlib
|
||||
import os
|
||||
|
||||
import unittest2
|
||||
|
||||
import openerp
|
||||
import openerp.tests.common
|
||||
|
||||
class test_ir_attachment(openerp.tests.common.TransactionCase):
|
||||
HASH_SPLIT = 2 # FIXME: testing implementations detail is not a good idea
|
||||
|
||||
def test_00_attachment_flow(self):
|
||||
class test_ir_attachment(openerp.tests.common.TransactionCase):
|
||||
def setUp(self):
|
||||
registry, cr, uid = self.registry, self.cr, self.uid
|
||||
root_path = openerp.tools.config['root_path']
|
||||
ira = registry('ir.attachment')
|
||||
self.ira = registry('ir.attachment')
|
||||
self.filestore = self.ira._filestore(cr, uid)
|
||||
|
||||
# Blob1
|
||||
blob1 = 'blob1'
|
||||
blob1_b64 = blob1.encode('base64')
|
||||
blob1_hash = hashlib.sha1(blob1).hexdigest()
|
||||
blob1_fname = blob1_hash[:3] + '/' + blob1_hash
|
||||
self.blob1 = 'blob1'
|
||||
self.blob1_b64 = self.blob1.encode('base64')
|
||||
blob1_hash = hashlib.sha1(self.blob1).hexdigest()
|
||||
self.blob1_fname = blob1_hash[:HASH_SPLIT] + '/' + blob1_hash
|
||||
|
||||
# Blob2
|
||||
blob2 = 'blob2'
|
||||
blob2_b64 = blob2.encode('base64')
|
||||
blob2_hash = hashlib.sha1(blob2).hexdigest()
|
||||
blob2_fname = blob2_hash[:3] + '/' + blob2_hash
|
||||
self.blob2_b64 = blob2.encode('base64')
|
||||
|
||||
def test_01_store_in_db(self):
|
||||
registry, cr, uid = self.registry, self.cr, self.uid
|
||||
|
||||
# force storing in database
|
||||
registry('ir.config_parameter').set_param(cr, uid, 'ir_attachment.location', '')
|
||||
|
||||
# 'ir_attachment.location' is undefined test database storage
|
||||
a1 = ira.create(cr, uid, {'name': 'a1', 'datas': blob1_b64})
|
||||
a1_read = ira.read(cr, uid, [a1], ['datas'])
|
||||
self.assertEqual(a1_read[0]['datas'], blob1_b64)
|
||||
a1 = self.ira.create(cr, uid, {'name': 'a1', 'datas': self.blob1_b64})
|
||||
a1_read = self.ira.read(cr, uid, [a1], ['datas'])
|
||||
self.assertEqual(a1_read[0]['datas'], self.blob1_b64)
|
||||
|
||||
cr.execute("select id,db_datas from ir_attachment where id = %s", (a1,) )
|
||||
a1_db_datas = str(cr.fetchall()[0][1])
|
||||
self.assertEqual(a1_db_datas, blob1_b64)
|
||||
a1_db_datas = self.ira.browse(cr, uid, a1).db_datas
|
||||
self.assertEqual(a1_db_datas, self.blob1_b64)
|
||||
|
||||
# define a location for filestore
|
||||
registry('ir.config_parameter').set_param(cr, uid, 'ir_attachment.location', 'file:///filestore')
|
||||
def test_02_store_on_disk(self):
|
||||
registry, cr, uid = self.registry, self.cr, self.uid
|
||||
|
||||
# Test file storage
|
||||
a2 = ira.create(cr, uid, {'name': 'a2', 'datas': blob1_b64})
|
||||
a2_read = ira.read(cr, uid, [a2], ['datas'])
|
||||
self.assertEqual(a2_read[0]['datas'], blob1_b64)
|
||||
a2 = self.ira.create(cr, uid, {'name': 'a2', 'datas': self.blob1_b64})
|
||||
a2_store_fname = self.ira.browse(cr, uid, a2).store_fname
|
||||
|
||||
cr.execute("select id,store_fname from ir_attachment where id = %s", (a2,) )
|
||||
a2_store_fname = cr.fetchall()[0][1]
|
||||
self.assertEqual(a2_store_fname, blob1_fname)
|
||||
self.assertEqual(a2_store_fname, self.blob1_fname)
|
||||
self.assertTrue(os.path.isdir(os.path.join(self.filestore, a2_store_fname)))
|
||||
|
||||
a2_fn = os.path.join(root_path, 'filestore', cr.dbname, blob1_hash[:3], blob1_hash)
|
||||
fc = file(a2_fn).read()
|
||||
self.assertEqual(fc, blob1)
|
||||
def test_03_no_duplication(self):
|
||||
registry, cr, uid = self.registry, self.cr, self.uid
|
||||
|
||||
# create a3 with same blob
|
||||
a3 = ira.create(cr, uid, {'name': 'a3', 'datas': blob1_b64})
|
||||
a3_read = ira.read(cr, uid, [a3], ['datas'])
|
||||
self.assertEqual(a3_read[0]['datas'], blob1_b64)
|
||||
a2 = self.ira.create(cr, uid, {'name': 'a2', 'datas': self.blob1_b64})
|
||||
a2_store_fname = self.ira.browse(cr, uid, a2).store_fname
|
||||
|
||||
a3 = self.ira.create(cr, uid, {'name': 'a3', 'datas': self.blob1_b64})
|
||||
a3_store_fname = self.ira.browse(cr, uid, a3).store_fname
|
||||
|
||||
cr.execute("select id,store_fname from ir_attachment where id = %s", (a3,) )
|
||||
a3_store_fname = cr.fetchall()[0][1]
|
||||
self.assertEqual(a3_store_fname, a2_store_fname)
|
||||
|
||||
# create a4 blob2
|
||||
a4 = ira.create(cr, uid, {'name': 'a4', 'datas': blob2_b64})
|
||||
a4_read = ira.read(cr, uid, [a4], ['datas'])
|
||||
self.assertEqual(a4_read[0]['datas'], blob2_b64)
|
||||
def test_04_keep_file(self):
|
||||
registry, cr, uid = self.registry, self.cr, self.uid
|
||||
|
||||
a4_fn = os.path.join(root_path, 'filestore', cr.dbname, blob2_hash[:3], blob2_hash)
|
||||
self.assertTrue(os.path.isfile(a4_fn))
|
||||
a2 = self.ira.create(cr, uid, {'name': 'a2', 'datas': self.blob1_b64})
|
||||
a3 = self.ira.create(cr, uid, {'name': 'a3', 'datas': self.blob1_b64})
|
||||
|
||||
# delete a3 but file stays
|
||||
ira.unlink(cr, uid, [a3])
|
||||
a2_store_fname = self.ira.browse(cr, uid, a2).store_fname
|
||||
a2_fn = os.path.join(self.filestore, a2_store_fname)
|
||||
|
||||
self.ira.unlink(cr, uid, [a3])
|
||||
self.assertTrue(os.path.isfile(a2_fn))
|
||||
|
||||
# delete a2 it is unlinked
|
||||
ira.unlink(cr, uid, [a2])
|
||||
self.ira.unlink(cr, uid, [a2])
|
||||
self.assertFalse(os.path.isfile(a2_fn))
|
||||
|
||||
# update a4 blob2 by blob1
|
||||
ira.write(cr, uid, [a4], {'datas': blob1_b64})
|
||||
a4_read = ira.read(cr, uid, [a4], ['datas'])
|
||||
self.assertEqual(a4_read[0]['datas'], blob1_b64)
|
||||
def test_05_change_data_change_file(self):
|
||||
registry, cr, uid = self.registry, self.cr, self.uid
|
||||
|
||||
a2 = self.ira.create(cr, uid, {'name': 'a2', 'datas': self.blob1_b64})
|
||||
a2_store_fname = self.ira.browse(cr, uid, a2).store_fname
|
||||
a2_fn = os.path.join(self.filestore, a2_store_fname)
|
||||
|
||||
# file of a4 disapear and a2 reappear
|
||||
self.assertFalse(os.path.isfile(a4_fn))
|
||||
self.assertTrue(os.path.isfile(a2_fn))
|
||||
|
||||
# everybody applause
|
||||
self.ira.write(cr, uid, [a2], {'datas': self.blob2_b64})
|
||||
self.assertFalse(os.path.isfile(a2_fn))
|
||||
|
||||
new_a2_store_fname = self.ira.browse(cr, uid, a2).store_fname
|
||||
self.assertNotEqual(a2_store_fname, new_a2_store_fname)
|
||||
|
||||
new_a2_fn = os.path.join(self.filestore, new_a2_store_fname)
|
||||
self.assertTrue(os.path.isfile(new_a2_fn))
|
||||
|
|
|
|||
|
|
@ -276,15 +276,10 @@ def exp_rename(old_name, new_name):
|
|||
cr.autocommit(True) # avoid transaction block
|
||||
try:
|
||||
cr.execute('ALTER DATABASE "%s" RENAME TO "%s"' % (old_name, new_name))
|
||||
_logger.info('RENAME DB: %s -> %s', old_name, new_name)
|
||||
except Exception, e:
|
||||
_logger.error('RENAME DB: %s -> %s failed:\n%s', old_name, new_name, e)
|
||||
raise Exception("Couldn't rename database %s to %s: %s" % (old_name, new_name, e))
|
||||
else:
|
||||
fs = os.path.join(openerp.tools.config['root_path'], 'filestore')
|
||||
if os.path.exists(os.path.join(fs, old_name)):
|
||||
os.rename(os.path.join(fs, old_name), os.path.join(fs, new_name))
|
||||
|
||||
_logger.info('RENAME DB: %s -> %s', old_name, new_name)
|
||||
return True
|
||||
|
||||
def exp_db_exist(db_name):
|
||||
|
|
|
|||
Loading…
Reference in New Issue