Compare commits
No commits in common. "04fe4e3a8349ef97785b4e27ef460d36b6ee4c29" and "6232f76561e78de4972b700e2d31efb5b232125a" have entirely different histories.
04fe4e3a83
...
6232f76561
92
README.md
92
README.md
@ -1,95 +1,3 @@
|
|||||||
# pytest
|
# pytest
|
||||||
|
|
||||||
Python Modul zur Auswertung von Versuchsdaten des Straßenbaus
|
Python Modul zur Auswertung von Versuchsdaten des Straßenbaus
|
||||||
|
|
||||||
## Übersicht
|
|
||||||
### Level 1: System Context diagram
|
|
||||||
|
|
||||||
|
|
||||||
## Struktur der Daten/Datenmodell
|
|
||||||
|
|
||||||
```mermaid
|
|
||||||
classDiagram
|
|
||||||
Project --> Workpackage
|
|
||||||
|
|
||||||
Organisation --> Material
|
|
||||||
Organisation --> Project
|
|
||||||
Organisation --> User
|
|
||||||
|
|
||||||
|
|
||||||
Material --> Sheartest
|
|
||||||
Material --> CITT
|
|
||||||
|
|
||||||
|
|
||||||
Sheartest --> ShearTestData
|
|
||||||
Sheartest <-- Project
|
|
||||||
Sheartest <.. Workpackage
|
|
||||||
|
|
||||||
CITT --> CITTData
|
|
||||||
CITT <-- Project
|
|
||||||
CITT <.. Workpackage
|
|
||||||
|
|
||||||
Project --> Material
|
|
||||||
|
|
||||||
User ..> Project
|
|
||||||
User ..> Material
|
|
||||||
User ..> Workpackage
|
|
||||||
User --> CITT
|
|
||||||
User --> Sheartest
|
|
||||||
|
|
||||||
Sheartest <-- Organisation
|
|
||||||
|
|
||||||
|
|
||||||
class Organisation{
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
class User {
|
|
||||||
orgID
|
|
||||||
}
|
|
||||||
|
|
||||||
class Project{
|
|
||||||
orgID: Organisation ID
|
|
||||||
(uID): User ID
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
class Workpackage{
|
|
||||||
pID: Project ID
|
|
||||||
(uID): User ID
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
class Material{
|
|
||||||
orgID: Organisation ID
|
|
||||||
[pID]: Project ID
|
|
||||||
(uID): User ID
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
class CITT {
|
|
||||||
mID: Material ID
|
|
||||||
pID: Project ID
|
|
||||||
(wIP): workpackage ID
|
|
||||||
(uID): User ID
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
class Sheartest {
|
|
||||||
orgID:
|
|
||||||
mID: Material ID
|
|
||||||
pID: Project ID
|
|
||||||
(wIP): workpackage ID
|
|
||||||
(uID): User ID
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
class ShearTestData {
|
|
||||||
shearID: Sheartest ID
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
class CITTData {
|
|
||||||
cittID: CITT ID
|
|
||||||
|
|
||||||
}
|
|
@ -1,5 +1,3 @@
|
|||||||
from .exceptions import HashInDbError
|
|
||||||
from .filehasher import calc_hash_of_file
|
from .filehasher import calc_hash_of_file
|
||||||
|
|
||||||
__all__ = ["calc_hash_of_file",
|
__all__ = ["calc_hash_of_file"]
|
||||||
"HashInDbError"]
|
|
||||||
|
@ -1,8 +0,0 @@
|
|||||||
# define Python user-defined exceptions
|
|
||||||
|
|
||||||
class HashInDbError(Exception):
|
|
||||||
def __init__(self, value):
|
|
||||||
self.parameter = value
|
|
||||||
|
|
||||||
def __str__(self):
|
|
||||||
return repr(self.parameter)
|
|
@ -1,54 +0,0 @@
|
|||||||
from minio import Minio
|
|
||||||
from minio.commonconfig import Tags
|
|
||||||
|
|
||||||
|
|
||||||
class MinioClient():
|
|
||||||
|
|
||||||
def __init__(self, url, access_key, secret_key, secure=True):
|
|
||||||
"""
|
|
||||||
return minio client
|
|
||||||
"""
|
|
||||||
|
|
||||||
self.client = Minio(url, access_key, secret_key, secure=secure)
|
|
||||||
|
|
||||||
def create_bucket(self, name):
|
|
||||||
"""
|
|
||||||
check bucket exits and create
|
|
||||||
"""
|
|
||||||
|
|
||||||
found = self.client.bucket_exists(name)
|
|
||||||
if not found:
|
|
||||||
self.client.make_bucket(name)
|
|
||||||
""" FIX: Add Tags
|
|
||||||
|
|
||||||
tag_obj = Tags.new_bucket_tags()
|
|
||||||
for key, i in tags.items():
|
|
||||||
tag_obj[key] = i
|
|
||||||
|
|
||||||
self.client.set_bucket_tags(name)
|
|
||||||
"""
|
|
||||||
|
|
||||||
def upload_file(
|
|
||||||
self,
|
|
||||||
bucket,
|
|
||||||
filename: str,
|
|
||||||
outfilename,
|
|
||||||
outpath='',
|
|
||||||
content_type="application/csv",
|
|
||||||
metadata: dict = {},
|
|
||||||
):
|
|
||||||
"""
|
|
||||||
upload file to bucket
|
|
||||||
"""
|
|
||||||
|
|
||||||
if len(outpath) > 0:
|
|
||||||
if not outpath[-1] == '/':
|
|
||||||
outpath += '/'
|
|
||||||
|
|
||||||
self.client.fput_object(
|
|
||||||
bucket,
|
|
||||||
outpath + outfilename,
|
|
||||||
filename,
|
|
||||||
content_type=content_type,
|
|
||||||
metadata=metadata,
|
|
||||||
)
|
|
@ -7,7 +7,6 @@ import matplotlib.pyplot as plt
|
|||||||
import numpy as np
|
import numpy as np
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
import scipy.fft as sfft
|
import scipy.fft as sfft
|
||||||
|
|
||||||
from pytestpavement.analysis import cosfunc
|
from pytestpavement.analysis import cosfunc
|
||||||
from pytestpavement.helper.filehasher import calc_hash_of_file
|
from pytestpavement.helper.filehasher import calc_hash_of_file
|
||||||
|
|
||||||
@ -285,23 +284,15 @@ class DataSineLoad():
|
|||||||
idx_diff = np.diff(d.index)
|
idx_diff = np.diff(d.index)
|
||||||
dt_mean = idx_diff.mean()
|
dt_mean = idx_diff.mean()
|
||||||
|
|
||||||
gaps = idx_diff > (4 * dt_mean)
|
gaps = idx_diff > (2 * dt_mean)
|
||||||
has_gaps = any(gaps)
|
has_gaps = any(gaps)
|
||||||
|
|
||||||
if has_gaps == False:
|
if has_gaps == False:
|
||||||
data_list.append(d)
|
data_list.append(d)
|
||||||
|
|
||||||
else:
|
else:
|
||||||
|
|
||||||
#FIX: GAP FINDING
|
|
||||||
data_list.append(d)
|
|
||||||
"""
|
|
||||||
print('has gaps')
|
|
||||||
print(gaps)
|
|
||||||
idx_gaps = (np.where(gaps)[0] - 1)[0]
|
idx_gaps = (np.where(gaps)[0] - 1)[0]
|
||||||
print(idx_gaps)
|
|
||||||
data_list.append(d.iloc[0:idx_gaps])
|
data_list.append(d.iloc[0:idx_gaps])
|
||||||
"""
|
|
||||||
|
|
||||||
#add self.
|
#add self.
|
||||||
if len(data_list) == 0:
|
if len(data_list) == 0:
|
||||||
@ -314,61 +305,17 @@ class DataSineLoad():
|
|||||||
#break
|
#break
|
||||||
|
|
||||||
def _select_data(self):
|
def _select_data(self):
|
||||||
"""
|
""" select N load cycles from original data """
|
||||||
select N load cycles from original data
|
|
||||||
(a): Based on window of TP-Asphalt
|
|
||||||
(b) last N cycles
|
|
||||||
|
|
||||||
"""
|
|
||||||
|
|
||||||
def sel_df(df, num=5):
|
def sel_df(df, num=5):
|
||||||
N = df['N'].unique()
|
N = df['N'].unique()
|
||||||
freq = float(df['f'].unique()[0])
|
|
||||||
|
|
||||||
# define cycles to select
|
if len(N) > num:
|
||||||
if freq == 10.0:
|
df_sel = df[(df['N'] >= N[-num - 1]) & (df['N'] <= N[-2])]
|
||||||
Nfrom = 98
|
return df_sel
|
||||||
Nto = 103
|
|
||||||
elif freq == 5.0:
|
|
||||||
Nfrom = 93
|
|
||||||
Nto = 97
|
|
||||||
elif freq == 3.0:
|
|
||||||
Nfrom = 43
|
|
||||||
Nto = 47
|
|
||||||
elif freq == 1.0:
|
|
||||||
Nfrom = 13
|
|
||||||
Nto = 17
|
|
||||||
elif freq == 0.3:
|
|
||||||
Nfrom = 8
|
|
||||||
Nto = 12
|
|
||||||
elif freq == 0.1:
|
|
||||||
Nfrom = 3
|
|
||||||
Nto = 7
|
|
||||||
else:
|
else:
|
||||||
Nfrom = None
|
ValueError(
|
||||||
Nto = None
|
'Number of load cycles smaller than selectect values')
|
||||||
|
|
||||||
|
|
||||||
if Nfrom != None:
|
|
||||||
if len(N) > Nto - Nfrom:
|
|
||||||
df_sel = df[(df['N'] >= Nfrom) & (df['N'] <= Nto)]
|
|
||||||
if self.debug:
|
|
||||||
print('select N', Nfrom, Nto, freq)
|
|
||||||
print(df_sel.head())
|
|
||||||
print(df_sel.tail())
|
|
||||||
|
|
||||||
|
|
||||||
return df_sel
|
|
||||||
else:
|
|
||||||
ValueError(
|
|
||||||
'Number of load cycles smaller than selectect values')
|
|
||||||
else:
|
|
||||||
if len(N) > num:
|
|
||||||
df_sel = df[(df['N'] >= N[-num - 1]) & (df['N'] <= N[-2])]
|
|
||||||
return df_sel
|
|
||||||
else:
|
|
||||||
ValueError(
|
|
||||||
'Number of load cycles smaller than selectect values')
|
|
||||||
|
|
||||||
if not isinstance(self.data, list):
|
if not isinstance(self.data, list):
|
||||||
if self.number_of_load_cycles_for_analysis > 1:
|
if self.number_of_load_cycles_for_analysis > 1:
|
||||||
|
@ -4,16 +4,12 @@ from csv import reader
|
|||||||
import matplotlib.pyplot as plt
|
import matplotlib.pyplot as plt
|
||||||
import numpy as np
|
import numpy as np
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
from bson import ObjectId
|
|
||||||
|
|
||||||
from pytestpavement.analysis.regression import fit_cos, fit_cos_eval
|
from pytestpavement.analysis.regression import fit_cos, fit_cos_eval
|
||||||
from pytestpavement.functions.citt import calc_nu
|
from pytestpavement.functions.citt import calc_nu
|
||||||
#from pytestpavement.helper.exceptions import HashInDbError
|
#from pytestpavement.helper.exceptions import HashInDbError
|
||||||
from pytestpavement.io.geosys import read_geosys
|
from pytestpavement.io.geosys import read_geosys
|
||||||
from pytestpavement.labtests.base import DataSineLoad
|
from pytestpavement.labtests.base import DataSineLoad
|
||||||
from pytestpavement.models.citt import (CITTSiffnessResults,
|
from pytestpavement.models.citt import CyclicIndirectTensileTest
|
||||||
CyclicIndirectTensileTest)
|
|
||||||
from pytestpavement.models.data import CITTSiffness
|
|
||||||
|
|
||||||
|
|
||||||
def check_timeindex(t, cycles: int, frequency: float, error: float = 10.0):
|
def check_timeindex(t, cycles: int, frequency: float, error: float = 10.0):
|
||||||
@ -59,7 +55,6 @@ class CittTest(DataSineLoad):
|
|||||||
self.frequency = frequency
|
self.frequency = frequency
|
||||||
self.sigma = sigma
|
self.sigma = sigma
|
||||||
self.meta = meta
|
self.meta = meta
|
||||||
self.debug = debug
|
|
||||||
|
|
||||||
# process file
|
# process file
|
||||||
self._run()
|
self._run()
|
||||||
@ -185,45 +180,7 @@ class CittTest(DataSineLoad):
|
|||||||
if not self.fit.empty:
|
if not self.fit.empty:
|
||||||
self.fit = self.fit.reset_index(
|
self.fit = self.fit.reset_index(
|
||||||
drop=True).set_index('idx_data').sort_index()
|
drop=True).set_index('idx_data').sort_index()
|
||||||
|
|
||||||
def save(self, org_id: ObjectId, project_id: ObjectId, material_id: ObjectId, user_id: ObjectId = None, meta: dict = {}, wp_id=None):
|
|
||||||
|
|
||||||
for idx_fit, fit in self.fit.iterrows():
|
|
||||||
data = self.data[idx_fit]
|
|
||||||
|
|
||||||
meta['filehash'] = self.filehash
|
|
||||||
meta['org_id'] = org_id
|
|
||||||
meta['project_id'] = project_id
|
|
||||||
meta['material'] = material_id
|
|
||||||
meta['user_id'] = user_id
|
|
||||||
|
|
||||||
#check if result in db
|
|
||||||
#n = CITTSiffness.objects(**meta).count()
|
|
||||||
#print(n)
|
|
||||||
|
|
||||||
# write data
|
|
||||||
data_dict = fit.to_dict()
|
|
||||||
data_dict.update(meta)
|
|
||||||
|
|
||||||
f = CITTSiffnessResults(**data_dict).save()
|
|
||||||
|
|
||||||
data_out = dict(
|
|
||||||
time = data.index,
|
|
||||||
F= list(data['F']),
|
|
||||||
N = list(data['N']),
|
|
||||||
s_hor_1 = list(data['s_hor_1']),
|
|
||||||
s_hor_2 = list(data['s_hor_2']),
|
|
||||||
s_hor_sum = list(data['s_hor_sum']),
|
|
||||||
s_piston = list(data['s_piston']),
|
|
||||||
)
|
|
||||||
|
|
||||||
g = CITTSiffness(
|
|
||||||
result=f.id,
|
|
||||||
**data_out
|
|
||||||
).save()
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
class CittTestTUDGeosys(CittTest):
|
class CittTestTUDGeosys(CittTest):
|
||||||
|
|
||||||
@ -327,7 +284,7 @@ class CittTestTUDTira(CittTest):
|
|||||||
'time', 'T', 'f', 'N', 'F', 's_hor_sum', 's_hor_1', 's_hor_2'
|
'time', 'T', 'f', 'N', 'F', 's_hor_sum', 's_hor_1', 's_hor_2'
|
||||||
]
|
]
|
||||||
# Header names after standardization; check if exists
|
# Header names after standardization; check if exists
|
||||||
self.val_header_names = ['speciment_diameter', 'speciment_diameter']
|
self.val_header_names = ['speciment_diameter']
|
||||||
|
|
||||||
self.columns_analyse = ['F', 's_hor_sum', 's_hor_1', 's_hor_2']
|
self.columns_analyse = ['F', 's_hor_sum', 's_hor_1', 's_hor_2']
|
||||||
|
|
||||||
@ -340,7 +297,7 @@ class CittTestTUDTira(CittTest):
|
|||||||
def _define_units(self):
|
def _define_units(self):
|
||||||
|
|
||||||
self.unit_s = 1. #ym in mm
|
self.unit_s = 1. #ym in mm
|
||||||
self.unit_F = -1.0 #N
|
self.unit_F = 1 #N
|
||||||
self.unit_t = 1. #s
|
self.unit_t = 1. #s
|
||||||
|
|
||||||
def _read_data(self, skiprows=28, hasunits=True):
|
def _read_data(self, skiprows=28, hasunits=True):
|
||||||
@ -351,6 +308,7 @@ class CittTestTUDTira(CittTest):
|
|||||||
meta = {}
|
meta = {}
|
||||||
|
|
||||||
splitsign = ':;'
|
splitsign = ':;'
|
||||||
|
|
||||||
with open(self.file, 'r', encoding=self.encoding) as f:
|
with open(self.file, 'r', encoding=self.encoding) as f:
|
||||||
count = 0
|
count = 0
|
||||||
|
|
||||||
@ -391,7 +349,7 @@ class CittTestTUDTira(CittTest):
|
|||||||
decimal=',',
|
decimal=',',
|
||||||
sep=';')
|
sep=';')
|
||||||
|
|
||||||
data = data.iloc[2:]
|
data = data.iloc[1:]
|
||||||
|
|
||||||
data.columns = head
|
data.columns = head
|
||||||
|
|
||||||
@ -456,26 +414,13 @@ class CittTestTUDTira(CittTest):
|
|||||||
|
|
||||||
self.data.columns = colnames
|
self.data.columns = colnames
|
||||||
|
|
||||||
def _calc_missiong_values(self):
|
def _calc_missiong_values(self):
|
||||||
|
|
||||||
cols = self.data.columns
|
cols = self.data.columns
|
||||||
|
|
||||||
if not 'f' in cols:
|
if not 's_hor_sum' in cols:
|
||||||
self.data['f'] = float(
|
self.data['s_hor_sum'] = self.data[['s_hor_1',
|
||||||
self.meta['Sollfrequnz'].split(';')[0].strip().replace(
|
's_hor_2']].sum(axis=1)
|
||||||
',', '.'))
|
|
||||||
|
|
||||||
if not 's_hor_sum' in cols:
|
|
||||||
self.data['s_hor_sum'] = self.data[['s_hor_1',
|
|
||||||
's_hor_2']].sum(axis=1)
|
|
||||||
|
|
||||||
self.sigma = float(
|
|
||||||
self.meta['Oberspannung'].split(';')[0].strip().replace(',', '.'))
|
|
||||||
self.temperature = float(
|
|
||||||
self.meta['Solltemperatur'].split(';')[0].strip().replace(
|
|
||||||
',', '.'))
|
|
||||||
self.frequency = float(
|
|
||||||
self.meta['Sollfrequnz'].split(';')[0].strip().replace(',', '.'))
|
|
||||||
|
|
||||||
|
|
||||||
class CittTestBASt(CittTest):
|
class CittTestBASt(CittTest):
|
||||||
|
@ -1,17 +1,13 @@
|
|||||||
import os
|
import os
|
||||||
import tempfile
|
|
||||||
|
|
||||||
import lmfit as lm
|
import lmfit as lm
|
||||||
import matplotlib.pyplot as plt
|
import matplotlib.pyplot as plt
|
||||||
import numpy as np
|
import numpy as np
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
import py7zr
|
|
||||||
import scipy.fft as sfft
|
import scipy.fft as sfft
|
||||||
import seaborn as sns
|
import seaborn as sns
|
||||||
|
|
||||||
from pytestpavement.analysis.regression import fit_cos, fit_cos_eval
|
from pytestpavement.analysis.regression import fit_cos, fit_cos_eval
|
||||||
from pytestpavement.io.geosys import read_geosys
|
from pytestpavement.io.geosys import read_geosys
|
||||||
from pytestpavement.io.minio import MinioClient
|
|
||||||
from pytestpavement.labtests.base import DataSineLoad
|
from pytestpavement.labtests.base import DataSineLoad
|
||||||
from pytestpavement.models.data import DataSheartest
|
from pytestpavement.models.data import DataSheartest
|
||||||
from pytestpavement.models.sheartest import DynamicShearTestExtension
|
from pytestpavement.models.sheartest import DynamicShearTestExtension
|
||||||
@ -26,17 +22,13 @@ class ShearTest(DataSineLoad):
|
|||||||
fname: str,
|
fname: str,
|
||||||
debug: bool = False,
|
debug: bool = False,
|
||||||
gap_width: float = 1.0,
|
gap_width: float = 1.0,
|
||||||
roundtemperature: bool = True,
|
roundtemperature: bool = True):
|
||||||
archive_file=False,
|
|
||||||
s3_params: dict = {}):
|
|
||||||
|
|
||||||
#set parameter
|
#set parameter
|
||||||
self.gap_width = gap_width
|
self.gap_width = gap_width
|
||||||
self.debug = debug
|
self.debug = debug
|
||||||
self.file = fname
|
self.file = fname
|
||||||
self.roundtemperature = roundtemperature
|
self.roundtemperature = roundtemperature
|
||||||
self.archive_file = archive_file
|
|
||||||
self.s3_params = s3_params
|
|
||||||
|
|
||||||
# process file
|
# process file
|
||||||
self._run()
|
self._run()
|
||||||
@ -167,7 +159,6 @@ class ShearTestExtension(ShearTest):
|
|||||||
f=fit['f'],
|
f=fit['f'],
|
||||||
sigma_normal=fit['sigma_normal'],
|
sigma_normal=fit['sigma_normal'],
|
||||||
T=fit['T'],
|
T=fit['T'],
|
||||||
|
|
||||||
extension=fit['extension'],
|
extension=fit['extension'],
|
||||||
material1=material1,
|
material1=material1,
|
||||||
material2=material2,
|
material2=material2,
|
||||||
@ -176,6 +167,17 @@ class ShearTestExtension(ShearTest):
|
|||||||
).count()
|
).count()
|
||||||
if n > 0: continue
|
if n > 0: continue
|
||||||
|
|
||||||
|
#save data
|
||||||
|
rdata = DataSheartest(
|
||||||
|
time=data.index.values,
|
||||||
|
F=data['F'].values,
|
||||||
|
N=data['N'].values,
|
||||||
|
s_vert_1=data['s_vert_1'].values,
|
||||||
|
s_vert_2=data['s_vert_2'].values,
|
||||||
|
s_vert_sum=data['s_vert_sum'].values,
|
||||||
|
s_piston=data['s_piston'].values,
|
||||||
|
).save()
|
||||||
|
|
||||||
# save fit
|
# save fit
|
||||||
|
|
||||||
values = {}
|
values = {}
|
||||||
@ -201,64 +203,15 @@ class ShearTestExtension(ShearTest):
|
|||||||
material2=material2,
|
material2=material2,
|
||||||
bounding=bounding,
|
bounding=bounding,
|
||||||
#results
|
#results
|
||||||
|
data=rdata,
|
||||||
stiffness=fit['G'],
|
stiffness=fit['G'],
|
||||||
#
|
#
|
||||||
**values).save()
|
**values).save()
|
||||||
|
|
||||||
#save raw data
|
|
||||||
rdata = DataSheartest(
|
|
||||||
result_id=r.id,
|
|
||||||
time=data.index.values,
|
|
||||||
F=data['F'].values,
|
|
||||||
N=data['N'].values,
|
|
||||||
s_vert_1=data['s_vert_1'].values,
|
|
||||||
s_vert_2=data['s_vert_2'].values,
|
|
||||||
s_vert_sum=data['s_vert_sum'].values,
|
|
||||||
s_piston=data['s_piston'].values,
|
|
||||||
).save()
|
|
||||||
|
|
||||||
except:
|
except:
|
||||||
print('error saving data')
|
print('error saving data')
|
||||||
raise
|
raise
|
||||||
rdata.delete()
|
rdata.delete()
|
||||||
|
|
||||||
if self.archive_file:
|
|
||||||
mclient = MinioClient(self.s3_params['S3_URL'],
|
|
||||||
self.s3_params['S3_ACCESS_KEY'],
|
|
||||||
self.s3_params['S3_SECRET_KEY'])
|
|
||||||
|
|
||||||
bucket = str(meta['org_id'])
|
|
||||||
mclient.create_bucket(bucket)
|
|
||||||
|
|
||||||
extension = os.path.splitext(self.file)[-1]
|
|
||||||
ofilename = self.filehash + extension
|
|
||||||
outpath = 'sheartest'
|
|
||||||
|
|
||||||
metadata_s3 = {
|
|
||||||
'project_id': str(meta['project_id']),·
|
|
||||||
'user_id': str(meta['user_id']),
|
|
||||||
'filename': os.path.split(self.file)[-1],
|
|
||||||
'speciment': meta['speciment_name']
|
|
||||||
}
|
|
||||||
|
|
||||||
#compress data to tmpfolder
|
|
||||||
with tempfile.TemporaryDirectory() as tmpdirname:
|
|
||||||
|
|
||||||
ofilename_compressed = ofilename + '.7z'
|
|
||||||
|
|
||||||
compressed_file = os.path.join(tmpdirname,
|
|
||||||
ofilename_compressed)
|
|
||||||
|
|
||||||
with py7zr.SevenZipFile(compressed_file, 'w') as archive:
|
|
||||||
archive.writeall(self.file)
|
|
||||||
|
|
||||||
mclient.upload_file(bucket,
|
|
||||||
compressed_file,
|
|
||||||
ofilename_compressed,
|
|
||||||
outpath=outpath,
|
|
||||||
content_type="application/raw",
|
|
||||||
metadata=metadata_s3)
|
|
||||||
|
|
||||||
def _set_parameter(self):
|
def _set_parameter(self):
|
||||||
|
|
||||||
self.split_data_based_on_parameter = [
|
self.split_data_based_on_parameter = [
|
||||||
|
@ -2,5 +2,4 @@ from .citt import *
|
|||||||
from .material import *
|
from .material import *
|
||||||
from .project import *
|
from .project import *
|
||||||
from .sheartest import *
|
from .sheartest import *
|
||||||
from .usermanagement import *
|
|
||||||
from .workpackage import *
|
from .workpackage import *
|
||||||
|
@ -2,10 +2,8 @@ import datetime
|
|||||||
|
|
||||||
from mongoengine import *
|
from mongoengine import *
|
||||||
|
|
||||||
|
from .data import CITTSiffness
|
||||||
from .material import Material
|
from .material import Material
|
||||||
from .project import Project
|
|
||||||
from .usermanagement import Organisation, User
|
|
||||||
from .workpackage import Workpackage
|
|
||||||
|
|
||||||
|
|
||||||
class CyclicIndirectTensileTest(Document):
|
class CyclicIndirectTensileTest(Document):
|
||||||
@ -16,23 +14,16 @@ class CyclicIndirectTensileTest(Document):
|
|||||||
}})
|
}})
|
||||||
|
|
||||||
standard = StringField(default='TP Asphalt Teil 24')
|
standard = StringField(default='TP Asphalt Teil 24')
|
||||||
|
|
||||||
org_id = LazyReferenceField(Organisation, required=True)
|
|
||||||
project_id = LazyReferenceField(Project, required=True)
|
|
||||||
workpackage_id = LazyReferenceField(Workpackage, required=False)
|
|
||||||
|
|
||||||
user_id = LazyReferenceField(User,
|
|
||||||
required=True,
|
|
||||||
reverse_delete_rule=DO_NOTHING)
|
|
||||||
|
|
||||||
material = LazyReferenceField(Material, required=True)
|
|
||||||
|
|
||||||
tags = ListField(StringField())
|
|
||||||
|
|
||||||
|
lab = StringField(default='TU Dresden', required=True)
|
||||||
|
auditor = StringField(default=None)
|
||||||
machine = StringField(default=None)
|
machine = StringField(default=None)
|
||||||
|
|
||||||
filehash = StringField(required=True)
|
filehash = StringField(required=True)
|
||||||
speciment_name = StringField()
|
|
||||||
|
project = StringField(required=True)
|
||||||
|
workpackage = StringField()
|
||||||
|
material = ReferenceField(Material, required=True)
|
||||||
|
|
||||||
meta = {
|
meta = {
|
||||||
'allow_inheritance': True,
|
'allow_inheritance': True,
|
||||||
@ -44,7 +35,7 @@ class CyclicIndirectTensileTest(Document):
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
class CITTSiffnessResults(CyclicIndirectTensileTest):
|
class CITTSiffness(CyclicIndirectTensileTest):
|
||||||
|
|
||||||
#metadata
|
#metadata
|
||||||
f_set = FloatField()
|
f_set = FloatField()
|
||||||
@ -54,6 +45,10 @@ class CITTSiffnessResults(CyclicIndirectTensileTest):
|
|||||||
N_from = IntField()
|
N_from = IntField()
|
||||||
N_to = IntField()
|
N_to = IntField()
|
||||||
|
|
||||||
|
data = LazyReferenceField(CITTSiffness,
|
||||||
|
required=True,
|
||||||
|
reverse_delete_rule=CASCADE)
|
||||||
|
|
||||||
#results
|
#results
|
||||||
stiffness = FloatField()
|
stiffness = FloatField()
|
||||||
nu = FloatField()
|
nu = FloatField()
|
||||||
|
@ -2,9 +2,6 @@ import datetime
|
|||||||
|
|
||||||
from mongoengine import *
|
from mongoengine import *
|
||||||
|
|
||||||
from .citt import CyclicIndirectTensileTest
|
|
||||||
from .sheartest import DynamicShearTest
|
|
||||||
|
|
||||||
|
|
||||||
class RawData(Document):
|
class RawData(Document):
|
||||||
|
|
||||||
@ -25,11 +22,6 @@ class RawData(Document):
|
|||||||
|
|
||||||
class DataSheartest(RawData):
|
class DataSheartest(RawData):
|
||||||
|
|
||||||
#results
|
|
||||||
result_id = LazyReferenceField(DynamicShearTest,
|
|
||||||
required=True,
|
|
||||||
reverse_delete_rule=CASCADE)
|
|
||||||
|
|
||||||
# data
|
# data
|
||||||
time = ListField(FloatField())
|
time = ListField(FloatField())
|
||||||
F = ListField(FloatField())
|
F = ListField(FloatField())
|
||||||
@ -42,10 +34,6 @@ class DataSheartest(RawData):
|
|||||||
|
|
||||||
class CITTSiffness(RawData):
|
class CITTSiffness(RawData):
|
||||||
|
|
||||||
result = LazyReferenceField(CyclicIndirectTensileTest,
|
|
||||||
required=True,
|
|
||||||
reverse_delete_rule=CASCADE)
|
|
||||||
|
|
||||||
# data
|
# data
|
||||||
time = ListField(FloatField())
|
time = ListField(FloatField())
|
||||||
F = ListField(FloatField())
|
F = ListField(FloatField())
|
||||||
|
@ -1,11 +1,7 @@
|
|||||||
import datetime
|
import datetime
|
||||||
from re import T
|
|
||||||
|
|
||||||
from mongoengine import *
|
from mongoengine import *
|
||||||
|
|
||||||
from .project import Project
|
|
||||||
from .usermanagement import Organisation, User
|
|
||||||
|
|
||||||
|
|
||||||
class Material(Document):
|
class Material(Document):
|
||||||
|
|
||||||
@ -14,20 +10,8 @@ class Material(Document):
|
|||||||
"step": "60"
|
"step": "60"
|
||||||
}})
|
}})
|
||||||
|
|
||||||
org_id = LazyReferenceField(Organisation,
|
|
||||||
required=True,
|
|
||||||
reverse_delete_rule=CASCADE)
|
|
||||||
|
|
||||||
project_id = ListField(LazyReferenceField(Project,
|
|
||||||
required=False,
|
|
||||||
reverse_delete_rule=CASCADE),
|
|
||||||
required=True)
|
|
||||||
|
|
||||||
user_id = LazyReferenceField(User,
|
|
||||||
required=False,
|
|
||||||
reverse_delete_rule=DO_NOTHING)
|
|
||||||
|
|
||||||
tags = ListField(StringField())
|
tags = ListField(StringField())
|
||||||
|
project = StringField(required=False)
|
||||||
norm = StringField(required=True, default='TP Asphalt Teil 24')
|
norm = StringField(required=True, default='TP Asphalt Teil 24')
|
||||||
|
|
||||||
meta = {
|
meta = {
|
||||||
@ -36,11 +20,7 @@ class Material(Document):
|
|||||||
'index_background': True,
|
'index_background': True,
|
||||||
'index_cls': False,
|
'index_cls': False,
|
||||||
'auto_create_index': True,
|
'auto_create_index': True,
|
||||||
'collection': 'materials',
|
'collection': 'materials'
|
||||||
'indexes': [
|
|
||||||
[("material", 1)],
|
|
||||||
[("name", 1)],
|
|
||||||
]
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -62,14 +42,6 @@ class Bitumen(Material):
|
|||||||
young_modulus = DictField()
|
young_modulus = DictField()
|
||||||
|
|
||||||
|
|
||||||
class Bitumenemulsion(Material):
|
|
||||||
|
|
||||||
name = StringField()
|
|
||||||
material = StringField()
|
|
||||||
|
|
||||||
young_modulus = DictField()
|
|
||||||
|
|
||||||
|
|
||||||
class Epoxy(Material):
|
class Epoxy(Material):
|
||||||
name = StringField()
|
name = StringField()
|
||||||
material = StringField()
|
material = StringField()
|
||||||
@ -77,10 +49,6 @@ class Epoxy(Material):
|
|||||||
young_modulus = DictField()
|
young_modulus = DictField()
|
||||||
|
|
||||||
|
|
||||||
class Kompaktasphalt(Material):
|
|
||||||
name = StringField()
|
|
||||||
|
|
||||||
|
|
||||||
class Dummy(Material):
|
class Dummy(Material):
|
||||||
|
|
||||||
name = StringField()
|
name = StringField()
|
||||||
|
@ -2,29 +2,18 @@ import datetime
|
|||||||
|
|
||||||
from mongoengine import *
|
from mongoengine import *
|
||||||
|
|
||||||
from .usermanagement import Organisation, User
|
|
||||||
|
|
||||||
|
|
||||||
class Project(Document):
|
class Project(Document):
|
||||||
|
|
||||||
name = StringField(required=True)
|
name = StringField(required=True)
|
||||||
name_short = StringField(required=False)
|
name_short = StringField(required=False)
|
||||||
project_id = StringField(required=True)
|
project_id = StringField(required=True)
|
||||||
client = StringField(required=False)
|
|
||||||
|
|
||||||
date = DateTimeField(default=datetime.datetime.now,
|
date = DateTimeField(default=datetime.datetime.now,
|
||||||
wtf_options={"render_kw": {
|
wtf_options={"render_kw": {
|
||||||
"step": "60"
|
"step": "60"
|
||||||
}})
|
}})
|
||||||
|
|
||||||
org_id = LazyReferenceField(Organisation,
|
|
||||||
required=True,
|
|
||||||
reverse_delete_rule=CASCADE)
|
|
||||||
|
|
||||||
user_id = LazyReferenceField(User,
|
|
||||||
required=False,
|
|
||||||
reverse_delete_rule=DO_NOTHING)
|
|
||||||
|
|
||||||
tags = ListField(StringField())
|
tags = ListField(StringField())
|
||||||
|
|
||||||
meta = {
|
meta = {
|
||||||
@ -33,10 +22,6 @@ class Project(Document):
|
|||||||
'index_background': True,
|
'index_background': True,
|
||||||
'index_cls': False,
|
'index_cls': False,
|
||||||
'auto_create_index': True,
|
'auto_create_index': True,
|
||||||
'collection': 'projects',
|
'collection': 'projects'
|
||||||
'indexes': [
|
|
||||||
[("name_short", 1)],
|
|
||||||
[("name", 1)],
|
|
||||||
[("project_id", 1)],
|
|
||||||
]
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -2,9 +2,9 @@ import datetime
|
|||||||
|
|
||||||
from mongoengine import *
|
from mongoengine import *
|
||||||
|
|
||||||
|
from .data import DataSheartest
|
||||||
from .material import Material
|
from .material import Material
|
||||||
from .project import Project
|
from .project import Project
|
||||||
from .usermanagement import Organisation, User
|
|
||||||
from .workpackage import Workpackage
|
from .workpackage import Workpackage
|
||||||
|
|
||||||
|
|
||||||
@ -15,13 +15,19 @@ class DynamicShearTest(Document):
|
|||||||
"step": "60"
|
"step": "60"
|
||||||
}})
|
}})
|
||||||
|
|
||||||
org_id = LazyReferenceField(Organisation, required=True)
|
tags = ListField(StringField())
|
||||||
project_id = LazyReferenceField(Project, required=True)
|
|
||||||
workpackage_id = LazyReferenceField(Workpackage, required=False)
|
|
||||||
|
|
||||||
user_id = LazyReferenceField(User,
|
standard = StringField(default='TP Asphalt Teil 24')
|
||||||
required=True,
|
|
||||||
reverse_delete_rule=DO_NOTHING)
|
lab = StringField(default='TU Dresden', required=True)
|
||||||
|
auditor = StringField(default=None)
|
||||||
|
machine = StringField(default=None)
|
||||||
|
|
||||||
|
filehash = StringField(required=True)
|
||||||
|
speciment_name = StringField()
|
||||||
|
|
||||||
|
project = LazyReferenceField(Project, required=True)
|
||||||
|
workpackage = LazyReferenceField(Workpackage, required=True)
|
||||||
|
|
||||||
material1 = LazyReferenceField(Material, required=True)
|
material1 = LazyReferenceField(Material, required=True)
|
||||||
material2 = LazyReferenceField(Material, required=True)
|
material2 = LazyReferenceField(Material, required=True)
|
||||||
@ -29,36 +35,13 @@ class DynamicShearTest(Document):
|
|||||||
|
|
||||||
gap_width = FloatField(default=1.0)
|
gap_width = FloatField(default=1.0)
|
||||||
|
|
||||||
tags = ListField(StringField())
|
|
||||||
|
|
||||||
standard = StringField(default='TP Asphalt Teil 24')
|
|
||||||
|
|
||||||
machine = StringField(default=None)
|
|
||||||
|
|
||||||
filehash = StringField(required=True)
|
|
||||||
speciment_name = StringField()
|
|
||||||
|
|
||||||
meta = {
|
meta = {
|
||||||
'allow_inheritance':
|
'allow_inheritance': True,
|
||||||
True,
|
|
||||||
'index_opts': {},
|
'index_opts': {},
|
||||||
'index_background':
|
'index_background': True,
|
||||||
True,
|
'index_cls': False,
|
||||||
'index_cls':
|
'auto_create_index': True,
|
||||||
False,
|
'collection': 'sheartest',
|
||||||
'auto_create_index':
|
|
||||||
True,
|
|
||||||
'collection':
|
|
||||||
'sheartest',
|
|
||||||
'indexes': [
|
|
||||||
[("lab", 1)],
|
|
||||||
[("speciment_name", 1)],
|
|
||||||
[("project", 1)],
|
|
||||||
[("bruch", 1)],
|
|
||||||
[("lab", 1), ("project", 1)],
|
|
||||||
[("lab", 1), ("project", 1), ("workpackage", 1)],
|
|
||||||
[("lab", 1), ("project", 1), ("bounding", 1)],
|
|
||||||
]
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -70,6 +53,11 @@ class DynamicShearTestExtension(DynamicShearTest):
|
|||||||
T = FloatField(required=True)
|
T = FloatField(required=True)
|
||||||
extension = FloatField(required=True)
|
extension = FloatField(required=True)
|
||||||
|
|
||||||
|
#results
|
||||||
|
data = LazyReferenceField(DataSheartest,
|
||||||
|
required=True,
|
||||||
|
reverse_delete_rule=CASCADE)
|
||||||
|
|
||||||
stiffness = FloatField(required=True)
|
stiffness = FloatField(required=True)
|
||||||
bruch = BooleanField(required=True)
|
bruch = BooleanField(required=True)
|
||||||
#fit parameter
|
#fit parameter
|
||||||
|
@ -1,48 +0,0 @@
|
|||||||
import datetime
|
|
||||||
|
|
||||||
from mongoengine import *
|
|
||||||
|
|
||||||
|
|
||||||
class Organisation(Document):
|
|
||||||
|
|
||||||
name = StringField(required=True)
|
|
||||||
name_short = StringField(required=True)
|
|
||||||
|
|
||||||
date = DateTimeField(default=datetime.datetime.now,
|
|
||||||
wtf_options={"render_kw": {
|
|
||||||
"step": "60"
|
|
||||||
}})
|
|
||||||
|
|
||||||
meta = {
|
|
||||||
'allow_inheritance': True,
|
|
||||||
'index_opts': {},
|
|
||||||
'index_background': True,
|
|
||||||
'index_cls': False,
|
|
||||||
'auto_create_index': True,
|
|
||||||
'collection': 'organisation'
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
class User(Document):
|
|
||||||
|
|
||||||
active = BooleanField(required=True, default=True)
|
|
||||||
|
|
||||||
org_id = LazyReferenceField(Organisation,
|
|
||||||
required=True,
|
|
||||||
reverse_delete_rule=CASCADE)
|
|
||||||
|
|
||||||
date_added = DateTimeField(default=datetime.datetime.now,
|
|
||||||
wtf_options={"render_kw": {
|
|
||||||
"step": "60"
|
|
||||||
}})
|
|
||||||
|
|
||||||
name = StringField(required=True)
|
|
||||||
|
|
||||||
meta = {
|
|
||||||
'allow_inheritance': True,
|
|
||||||
'index_opts': {},
|
|
||||||
'index_background': True,
|
|
||||||
'index_cls': False,
|
|
||||||
'auto_create_index': True,
|
|
||||||
'collection': 'user'
|
|
||||||
}
|
|
@ -3,26 +3,21 @@ import datetime
|
|||||||
from mongoengine import *
|
from mongoengine import *
|
||||||
|
|
||||||
from .project import Project
|
from .project import Project
|
||||||
from .usermanagement import User
|
|
||||||
|
|
||||||
|
|
||||||
class Workpackage(Document):
|
class Workpackage(Document):
|
||||||
|
|
||||||
name = StringField(required=True)
|
name = StringField(required=True)
|
||||||
name_short = StringField(required=False)
|
name_short = StringField(required=False)
|
||||||
wp_id = StringField(required=True)
|
wp_id = StringField(required=True)
|
||||||
|
|
||||||
project_id = LazyReferenceField(Project, required=True)
|
project = LazyReferenceField(Project, required=True)
|
||||||
|
|
||||||
user_id = LazyReferenceField(User,
|
|
||||||
required=False,
|
|
||||||
reverse_delete_rule=DO_NOTHING)
|
|
||||||
|
|
||||||
date = DateTimeField(default=datetime.datetime.now,
|
date = DateTimeField(default=datetime.datetime.now,
|
||||||
wtf_options={"render_kw": {
|
wtf_options={"render_kw": {
|
||||||
"step": "60"
|
"step": "60"
|
||||||
}})
|
}})
|
||||||
|
|
||||||
meta = {
|
meta = {
|
||||||
'allow_inheritance': True,
|
'allow_inheritance': True,
|
||||||
'index_opts': {},
|
'index_opts': {},
|
||||||
|
Loading…
Reference in New Issue
Block a user