diff --git a/README.md b/README.md index 52e7f1e..c15ce77 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,95 @@ # pytest Python Modul zur Auswertung von Versuchsdaten des Straßenbaus + +## Übersicht +### Level 1: System Context diagram + + +## Struktur der Daten/Datenmodell + +```mermaid +classDiagram + Project --> Workpackage + + Organisation --> Material + Organisation --> Project + Organisation --> User + + + Material --> Sheartest + Material --> CITT + + + Sheartest --> ShearTestData + Sheartest <-- Project + Sheartest <.. Workpackage + + CITT --> CITTData + CITT <-- Project + CITT <.. Workpackage + + Project --> Material + + User ..> Project + User ..> Material + User ..> Workpackage + User --> CITT + User --> Sheartest + + Sheartest <-- Organisation + + + class Organisation{ + + } + + class User { + orgID + } + + class Project{ + orgID: Organisation ID + (uID): User ID + + } + + class Workpackage{ + pID: Project ID + (uID): User ID + + } + + class Material{ + orgID: Organisation ID + [pID]: Project ID + (uID): User ID + + } + + class CITT { + mID: Material ID + pID: Project ID + (wIP): workpackage ID + (uID): User ID + + } + + class Sheartest { + orgID: + mID: Material ID + pID: Project ID + (wIP): workpackage ID + (uID): User ID + + } + + class ShearTestData { + shearID: Sheartest ID + + } + + class CITTData { + cittID: CITT ID + + } \ No newline at end of file diff --git a/src/pytestpavement/helper/__init__.py b/src/pytestpavement/helper/__init__.py index 37a666b..038caea 100644 --- a/src/pytestpavement/helper/__init__.py +++ b/src/pytestpavement/helper/__init__.py @@ -1,3 +1,5 @@ +from .exceptions import HashInDbError from .filehasher import calc_hash_of_file -__all__ = ["calc_hash_of_file"] +__all__ = ["calc_hash_of_file", + "HashInDbError"] diff --git a/src/pytestpavement/helper/exceptions.py b/src/pytestpavement/helper/exceptions.py new file mode 100644 index 0000000..f3b14cc --- /dev/null +++ b/src/pytestpavement/helper/exceptions.py @@ -0,0 +1,8 @@ +# define Python user-defined exceptions + +class HashInDbError(Exception): + def __init__(self, value): + self.parameter = value + + def __str__(self): + return repr(self.parameter) diff --git a/src/pytestpavement/labtests/base.py b/src/pytestpavement/labtests/base.py index 095df75..c8eb974 100644 --- a/src/pytestpavement/labtests/base.py +++ b/src/pytestpavement/labtests/base.py @@ -7,6 +7,7 @@ import matplotlib.pyplot as plt import numpy as np import pandas as pd import scipy.fft as sfft + from pytestpavement.analysis import cosfunc from pytestpavement.helper.filehasher import calc_hash_of_file @@ -284,15 +285,23 @@ class DataSineLoad(): idx_diff = np.diff(d.index) dt_mean = idx_diff.mean() - gaps = idx_diff > (2 * dt_mean) + gaps = idx_diff > (4 * dt_mean) has_gaps = any(gaps) if has_gaps == False: data_list.append(d) else: + + #FIX: GAP FINDING + data_list.append(d) + """ + print('has gaps') + print(gaps) idx_gaps = (np.where(gaps)[0] - 1)[0] + print(idx_gaps) data_list.append(d.iloc[0:idx_gaps]) + """ #add self. if len(data_list) == 0: @@ -305,17 +314,61 @@ class DataSineLoad(): #break def _select_data(self): - """ select N load cycles from original data """ + """ + select N load cycles from original data + (a): Based on window of TP-Asphalt + (b) last N cycles + + """ def sel_df(df, num=5): N = df['N'].unique() + freq = float(df['f'].unique()[0]) - if len(N) > num: - df_sel = df[(df['N'] >= N[-num - 1]) & (df['N'] <= N[-2])] - return df_sel + # define cycles to select + if freq == 10.0: + Nfrom = 98 + Nto = 103 + elif freq == 5.0: + Nfrom = 93 + Nto = 97 + elif freq == 3.0: + Nfrom = 43 + Nto = 47 + elif freq == 1.0: + Nfrom = 13 + Nto = 17 + elif freq == 0.3: + Nfrom = 8 + Nto = 12 + elif freq == 0.1: + Nfrom = 3 + Nto = 7 else: - ValueError( - 'Number of load cycles smaller than selectect values') + Nfrom = None + Nto = None + + + if Nfrom != None: + if len(N) > Nto - Nfrom: + df_sel = df[(df['N'] >= Nfrom) & (df['N'] <= Nto)] + if self.debug: + print('select N', Nfrom, Nto, freq) + print(df_sel.head()) + print(df_sel.tail()) + + + return df_sel + else: + ValueError( + 'Number of load cycles smaller than selectect values') + else: + if len(N) > num: + df_sel = df[(df['N'] >= N[-num - 1]) & (df['N'] <= N[-2])] + return df_sel + else: + ValueError( + 'Number of load cycles smaller than selectect values') if not isinstance(self.data, list): if self.number_of_load_cycles_for_analysis > 1: diff --git a/src/pytestpavement/labtests/citt.py b/src/pytestpavement/labtests/citt.py index b00663e..15b7121 100644 --- a/src/pytestpavement/labtests/citt.py +++ b/src/pytestpavement/labtests/citt.py @@ -4,12 +4,16 @@ from csv import reader import matplotlib.pyplot as plt import numpy as np import pandas as pd +from bson import ObjectId + from pytestpavement.analysis.regression import fit_cos, fit_cos_eval from pytestpavement.functions.citt import calc_nu #from pytestpavement.helper.exceptions import HashInDbError from pytestpavement.io.geosys import read_geosys from pytestpavement.labtests.base import DataSineLoad -from pytestpavement.models.citt import CyclicIndirectTensileTest +from pytestpavement.models.citt import (CITTSiffnessResults, + CyclicIndirectTensileTest) +from pytestpavement.models.data import CITTSiffness def check_timeindex(t, cycles: int, frequency: float, error: float = 10.0): @@ -55,6 +59,7 @@ class CittTest(DataSineLoad): self.frequency = frequency self.sigma = sigma self.meta = meta + self.debug = debug # process file self._run() @@ -180,7 +185,45 @@ class CittTest(DataSineLoad): if not self.fit.empty: self.fit = self.fit.reset_index( drop=True).set_index('idx_data').sort_index() + + def save(self, org_id: ObjectId, project_id: ObjectId, material_id: ObjectId, user_id: ObjectId = None, meta: dict = {}, wp_id=None): + for idx_fit, fit in self.fit.iterrows(): + data = self.data[idx_fit] + + meta['filehash'] = self.filehash + meta['org_id'] = org_id + meta['project_id'] = project_id + meta['material'] = material_id + meta['user_id'] = user_id + + #check if result in db + #n = CITTSiffness.objects(**meta).count() + #print(n) + + # write data + data_dict = fit.to_dict() + data_dict.update(meta) + + f = CITTSiffnessResults(**data_dict).save() + + data_out = dict( + time = data.index, + F= list(data['F']), + N = list(data['N']), + s_hor_1 = list(data['s_hor_1']), + s_hor_2 = list(data['s_hor_2']), + s_hor_sum = list(data['s_hor_sum']), + s_piston = list(data['s_piston']), + ) + + g = CITTSiffness( + result=f.id, + **data_out + ).save() + + + class CittTestTUDGeosys(CittTest): @@ -284,7 +327,7 @@ class CittTestTUDTira(CittTest): 'time', 'T', 'f', 'N', 'F', 's_hor_sum', 's_hor_1', 's_hor_2' ] # Header names after standardization; check if exists - self.val_header_names = ['speciment_diameter'] + self.val_header_names = ['speciment_diameter', 'speciment_diameter'] self.columns_analyse = ['F', 's_hor_sum', 's_hor_1', 's_hor_2'] @@ -297,7 +340,7 @@ class CittTestTUDTira(CittTest): def _define_units(self): self.unit_s = 1. #ym in mm - self.unit_F = 1 #N + self.unit_F = -1.0 #N self.unit_t = 1. #s def _read_data(self, skiprows=28, hasunits=True): @@ -308,7 +351,6 @@ class CittTestTUDTira(CittTest): meta = {} splitsign = ':;' - with open(self.file, 'r', encoding=self.encoding) as f: count = 0 @@ -349,7 +391,7 @@ class CittTestTUDTira(CittTest): decimal=',', sep=';') - data = data.iloc[1:] + data = data.iloc[2:] data.columns = head @@ -414,13 +456,26 @@ class CittTestTUDTira(CittTest): self.data.columns = colnames - def _calc_missiong_values(self): + def _calc_missiong_values(self): - cols = self.data.columns + cols = self.data.columns - if not 's_hor_sum' in cols: - self.data['s_hor_sum'] = self.data[['s_hor_1', - 's_hor_2']].sum(axis=1) + if not 'f' in cols: + self.data['f'] = float( + self.meta['Sollfrequnz'].split(';')[0].strip().replace( + ',', '.')) + + if not 's_hor_sum' in cols: + self.data['s_hor_sum'] = self.data[['s_hor_1', + 's_hor_2']].sum(axis=1) + + self.sigma = float( + self.meta['Oberspannung'].split(';')[0].strip().replace(',', '.')) + self.temperature = float( + self.meta['Solltemperatur'].split(';')[0].strip().replace( + ',', '.')) + self.frequency = float( + self.meta['Sollfrequnz'].split(';')[0].strip().replace(',', '.')) class CittTestBASt(CittTest): diff --git a/src/pytestpavement/labtests/sheartest.py b/src/pytestpavement/labtests/sheartest.py index 21332cd..98ff0e9 100644 --- a/src/pytestpavement/labtests/sheartest.py +++ b/src/pytestpavement/labtests/sheartest.py @@ -1,13 +1,17 @@ import os +import tempfile import lmfit as lm import matplotlib.pyplot as plt import numpy as np import pandas as pd +import py7zr import scipy.fft as sfft import seaborn as sns + from pytestpavement.analysis.regression import fit_cos, fit_cos_eval from pytestpavement.io.geosys import read_geosys +from pytestpavement.io.minio import MinioClient from pytestpavement.labtests.base import DataSineLoad from pytestpavement.models.data import DataSheartest from pytestpavement.models.sheartest import DynamicShearTestExtension @@ -22,13 +26,17 @@ class ShearTest(DataSineLoad): fname: str, debug: bool = False, gap_width: float = 1.0, - roundtemperature: bool = True): + roundtemperature: bool = True, + archive_file=False, + s3_params: dict = {}): #set parameter self.gap_width = gap_width self.debug = debug self.file = fname self.roundtemperature = roundtemperature + self.archive_file = archive_file + self.s3_params = s3_params # process file self._run() @@ -159,6 +167,7 @@ class ShearTestExtension(ShearTest): f=fit['f'], sigma_normal=fit['sigma_normal'], T=fit['T'], + extension=fit['extension'], material1=material1, material2=material2, @@ -167,17 +176,6 @@ class ShearTestExtension(ShearTest): ).count() if n > 0: continue - #save data - rdata = DataSheartest( - time=data.index.values, - F=data['F'].values, - N=data['N'].values, - s_vert_1=data['s_vert_1'].values, - s_vert_2=data['s_vert_2'].values, - s_vert_sum=data['s_vert_sum'].values, - s_piston=data['s_piston'].values, - ).save() - # save fit values = {} @@ -203,15 +201,64 @@ class ShearTestExtension(ShearTest): material2=material2, bounding=bounding, #results - data=rdata, stiffness=fit['G'], # **values).save() + + #save raw data + rdata = DataSheartest( + result_id=r.id, + time=data.index.values, + F=data['F'].values, + N=data['N'].values, + s_vert_1=data['s_vert_1'].values, + s_vert_2=data['s_vert_2'].values, + s_vert_sum=data['s_vert_sum'].values, + s_piston=data['s_piston'].values, + ).save() + except: print('error saving data') raise rdata.delete() + if self.archive_file: + mclient = MinioClient(self.s3_params['S3_URL'], + self.s3_params['S3_ACCESS_KEY'], + self.s3_params['S3_SECRET_KEY']) + + bucket = str(meta['org_id']) + mclient.create_bucket(bucket) + + extension = os.path.splitext(self.file)[-1] + ofilename = self.filehash + extension + outpath = 'sheartest' + + metadata_s3 = { + 'project_id': str(meta['project_id']),· + 'user_id': str(meta['user_id']), + 'filename': os.path.split(self.file)[-1], + 'speciment': meta['speciment_name'] + } + + #compress data to tmpfolder + with tempfile.TemporaryDirectory() as tmpdirname: + + ofilename_compressed = ofilename + '.7z' + + compressed_file = os.path.join(tmpdirname, + ofilename_compressed) + + with py7zr.SevenZipFile(compressed_file, 'w') as archive: + archive.writeall(self.file) + + mclient.upload_file(bucket, + compressed_file, + ofilename_compressed, + outpath=outpath, + content_type="application/raw", + metadata=metadata_s3) + def _set_parameter(self): self.split_data_based_on_parameter = [ diff --git a/src/pytestpavement/models/__init__.py b/src/pytestpavement/models/__init__.py index 7c5b27f..9250570 100644 --- a/src/pytestpavement/models/__init__.py +++ b/src/pytestpavement/models/__init__.py @@ -2,4 +2,5 @@ from .citt import * from .material import * from .project import * from .sheartest import * +from .usermanagement import * from .workpackage import * diff --git a/src/pytestpavement/models/citt.py b/src/pytestpavement/models/citt.py index 693a018..5ed4267 100644 --- a/src/pytestpavement/models/citt.py +++ b/src/pytestpavement/models/citt.py @@ -2,8 +2,10 @@ import datetime from mongoengine import * -from .data import CITTSiffness from .material import Material +from .project import Project +from .usermanagement import Organisation, User +from .workpackage import Workpackage class CyclicIndirectTensileTest(Document): @@ -14,16 +16,23 @@ class CyclicIndirectTensileTest(Document): }}) standard = StringField(default='TP Asphalt Teil 24') + + org_id = LazyReferenceField(Organisation, required=True) + project_id = LazyReferenceField(Project, required=True) + workpackage_id = LazyReferenceField(Workpackage, required=False) + + user_id = LazyReferenceField(User, + required=True, + reverse_delete_rule=DO_NOTHING) + + material = LazyReferenceField(Material, required=True) + + tags = ListField(StringField()) - lab = StringField(default='TU Dresden', required=True) - auditor = StringField(default=None) machine = StringField(default=None) filehash = StringField(required=True) - - project = StringField(required=True) - workpackage = StringField() - material = ReferenceField(Material, required=True) + speciment_name = StringField() meta = { 'allow_inheritance': True, @@ -35,7 +44,7 @@ class CyclicIndirectTensileTest(Document): } -class CITTSiffness(CyclicIndirectTensileTest): +class CITTSiffnessResults(CyclicIndirectTensileTest): #metadata f_set = FloatField() @@ -45,10 +54,6 @@ class CITTSiffness(CyclicIndirectTensileTest): N_from = IntField() N_to = IntField() - data = LazyReferenceField(CITTSiffness, - required=True, - reverse_delete_rule=CASCADE) - #results stiffness = FloatField() nu = FloatField() diff --git a/src/pytestpavement/models/data.py b/src/pytestpavement/models/data.py index 2c95043..01c8179 100644 --- a/src/pytestpavement/models/data.py +++ b/src/pytestpavement/models/data.py @@ -2,6 +2,9 @@ import datetime from mongoengine import * +from .citt import CyclicIndirectTensileTest +from .sheartest import DynamicShearTest + class RawData(Document): @@ -22,6 +25,11 @@ class RawData(Document): class DataSheartest(RawData): + #results + result_id = LazyReferenceField(DynamicShearTest, + required=True, + reverse_delete_rule=CASCADE) + # data time = ListField(FloatField()) F = ListField(FloatField()) @@ -34,6 +42,10 @@ class DataSheartest(RawData): class CITTSiffness(RawData): + result = LazyReferenceField(CyclicIndirectTensileTest, + required=True, + reverse_delete_rule=CASCADE) + # data time = ListField(FloatField()) F = ListField(FloatField()) diff --git a/src/pytestpavement/models/material.py b/src/pytestpavement/models/material.py index 40803f4..98c75df 100644 --- a/src/pytestpavement/models/material.py +++ b/src/pytestpavement/models/material.py @@ -1,7 +1,11 @@ import datetime +from re import T from mongoengine import * +from .project import Project +from .usermanagement import Organisation, User + class Material(Document): @@ -10,8 +14,20 @@ class Material(Document): "step": "60" }}) + org_id = LazyReferenceField(Organisation, + required=True, + reverse_delete_rule=CASCADE) + + project_id = ListField(LazyReferenceField(Project, + required=False, + reverse_delete_rule=CASCADE), + required=True) + + user_id = LazyReferenceField(User, + required=False, + reverse_delete_rule=DO_NOTHING) + tags = ListField(StringField()) - project = StringField(required=False) norm = StringField(required=True, default='TP Asphalt Teil 24') meta = { @@ -20,7 +36,11 @@ class Material(Document): 'index_background': True, 'index_cls': False, 'auto_create_index': True, - 'collection': 'materials' + 'collection': 'materials', + 'indexes': [ + [("material", 1)], + [("name", 1)], + ] } @@ -42,6 +62,14 @@ class Bitumen(Material): young_modulus = DictField() +class Bitumenemulsion(Material): + + name = StringField() + material = StringField() + + young_modulus = DictField() + + class Epoxy(Material): name = StringField() material = StringField() @@ -49,6 +77,10 @@ class Epoxy(Material): young_modulus = DictField() +class Kompaktasphalt(Material): + name = StringField() + + class Dummy(Material): name = StringField() diff --git a/src/pytestpavement/models/project.py b/src/pytestpavement/models/project.py index a3531d3..fe82cc2 100644 --- a/src/pytestpavement/models/project.py +++ b/src/pytestpavement/models/project.py @@ -2,18 +2,29 @@ import datetime from mongoengine import * +from .usermanagement import Organisation, User + class Project(Document): - + name = StringField(required=True) name_short = StringField(required=False) project_id = StringField(required=True) + client = StringField(required=False) date = DateTimeField(default=datetime.datetime.now, wtf_options={"render_kw": { "step": "60" }}) + org_id = LazyReferenceField(Organisation, + required=True, + reverse_delete_rule=CASCADE) + + user_id = LazyReferenceField(User, + required=False, + reverse_delete_rule=DO_NOTHING) + tags = ListField(StringField()) meta = { @@ -22,6 +33,10 @@ class Project(Document): 'index_background': True, 'index_cls': False, 'auto_create_index': True, - 'collection': 'projects' + 'collection': 'projects', + 'indexes': [ + [("name_short", 1)], + [("name", 1)], + [("project_id", 1)], + ] } - diff --git a/src/pytestpavement/models/sheartest.py b/src/pytestpavement/models/sheartest.py index 2a54549..ea4b486 100644 --- a/src/pytestpavement/models/sheartest.py +++ b/src/pytestpavement/models/sheartest.py @@ -2,9 +2,9 @@ import datetime from mongoengine import * -from .data import DataSheartest from .material import Material from .project import Project +from .usermanagement import Organisation, User from .workpackage import Workpackage @@ -15,19 +15,13 @@ class DynamicShearTest(Document): "step": "60" }}) - tags = ListField(StringField()) + org_id = LazyReferenceField(Organisation, required=True) + project_id = LazyReferenceField(Project, required=True) + workpackage_id = LazyReferenceField(Workpackage, required=False) - standard = StringField(default='TP Asphalt Teil 24') - - lab = StringField(default='TU Dresden', required=True) - auditor = StringField(default=None) - machine = StringField(default=None) - - filehash = StringField(required=True) - speciment_name = StringField() - - project = LazyReferenceField(Project, required=True) - workpackage = LazyReferenceField(Workpackage, required=True) + user_id = LazyReferenceField(User, + required=True, + reverse_delete_rule=DO_NOTHING) material1 = LazyReferenceField(Material, required=True) material2 = LazyReferenceField(Material, required=True) @@ -35,13 +29,36 @@ class DynamicShearTest(Document): gap_width = FloatField(default=1.0) + tags = ListField(StringField()) + + standard = StringField(default='TP Asphalt Teil 24') + + machine = StringField(default=None) + + filehash = StringField(required=True) + speciment_name = StringField() + meta = { - 'allow_inheritance': True, + 'allow_inheritance': + True, 'index_opts': {}, - 'index_background': True, - 'index_cls': False, - 'auto_create_index': True, - 'collection': 'sheartest', + 'index_background': + True, + 'index_cls': + False, + 'auto_create_index': + True, + 'collection': + 'sheartest', + 'indexes': [ + [("lab", 1)], + [("speciment_name", 1)], + [("project", 1)], + [("bruch", 1)], + [("lab", 1), ("project", 1)], + [("lab", 1), ("project", 1), ("workpackage", 1)], + [("lab", 1), ("project", 1), ("bounding", 1)], + ] } @@ -53,11 +70,6 @@ class DynamicShearTestExtension(DynamicShearTest): T = FloatField(required=True) extension = FloatField(required=True) - #results - data = LazyReferenceField(DataSheartest, - required=True, - reverse_delete_rule=CASCADE) - stiffness = FloatField(required=True) bruch = BooleanField(required=True) #fit parameter diff --git a/src/pytestpavement/models/usermanagement.py b/src/pytestpavement/models/usermanagement.py new file mode 100644 index 0000000..8e850c9 --- /dev/null +++ b/src/pytestpavement/models/usermanagement.py @@ -0,0 +1,48 @@ +import datetime + +from mongoengine import * + + +class Organisation(Document): + + name = StringField(required=True) + name_short = StringField(required=True) + + date = DateTimeField(default=datetime.datetime.now, + wtf_options={"render_kw": { + "step": "60" + }}) + + meta = { + 'allow_inheritance': True, + 'index_opts': {}, + 'index_background': True, + 'index_cls': False, + 'auto_create_index': True, + 'collection': 'organisation' + } + + +class User(Document): + + active = BooleanField(required=True, default=True) + + org_id = LazyReferenceField(Organisation, + required=True, + reverse_delete_rule=CASCADE) + + date_added = DateTimeField(default=datetime.datetime.now, + wtf_options={"render_kw": { + "step": "60" + }}) + + name = StringField(required=True) + + meta = { + 'allow_inheritance': True, + 'index_opts': {}, + 'index_background': True, + 'index_cls': False, + 'auto_create_index': True, + 'collection': 'user' + } \ No newline at end of file diff --git a/src/pytestpavement/models/workpackage.py b/src/pytestpavement/models/workpackage.py index c678d57..16040b6 100644 --- a/src/pytestpavement/models/workpackage.py +++ b/src/pytestpavement/models/workpackage.py @@ -3,21 +3,26 @@ import datetime from mongoengine import * from .project import Project +from .usermanagement import User class Workpackage(Document): - + name = StringField(required=True) name_short = StringField(required=False) wp_id = StringField(required=True) - - project = LazyReferenceField(Project, required=True) - + + project_id = LazyReferenceField(Project, required=True) + + user_id = LazyReferenceField(User, + required=False, + reverse_delete_rule=DO_NOTHING) + date = DateTimeField(default=datetime.datetime.now, - wtf_options={"render_kw": { - "step": "60" - }}) - + wtf_options={"render_kw": { + "step": "60" + }}) + meta = { 'allow_inheritance': True, 'index_opts': {},