Arbeit/pytest
1
0
Fork 0
Code Issues 2 Pull Requests Packages Projects Releases Activity

Datenstrukturen angepasst, Auswertung CITT Steifigkeit abgeschlossen

Browse Source
This commit is contained in:
Markus Clauß 2023-02-07 16:03:39 +01:00
parent 6232f76561
commit 05a16fa980
14 changed files with 466 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

92
README.md
View File

@ -1,3 +1,95 @@
# pytest
Python Modul zur Auswertung von Versuchsdaten des Straßenbaus
## Übersicht
### Level 1: System Context diagram
## Struktur der Daten/Datenmodell
```mermaid
classDiagram
Project --> Workpackage
Organisation --> Material
Organisation --> Project
Organisation --> User
Material --> Sheartest
Material --> CITT
Sheartest --> ShearTestData
Sheartest <-- Project
Sheartest <.. Workpackage
CITT --> CITTData
CITT <-- Project
CITT <.. Workpackage
Project --> Material
User ..> Project
User ..> Material
User ..> Workpackage
User --> CITT
User --> Sheartest
Sheartest <-- Organisation
class Organisation{
}
class User {
orgID
}
class Project{
orgID: Organisation ID
(uID): User ID
}
class Workpackage{
pID: Project ID
(uID): User ID
}
class Material{
orgID: Organisation ID
[pID]: Project ID
(uID): User ID
}
class CITT {
mID: Material ID
pID: Project ID
(wIP): workpackage ID
(uID): User ID
}
class Sheartest {
orgID:
mID: Material ID
pID: Project ID
(wIP): workpackage ID
(uID): User ID
}
class ShearTestData {
shearID: Sheartest ID
}
class CITTData {
cittID: CITT ID
}

4
src/pytestpavement/helper/__init__.py
View File

@ -1,3 +1,5 @@
from .exceptions import HashInDbError
from .filehasher import calc_hash_of_file
__all__ = ["calc_hash_of_file"]
__all__ = ["calc_hash_of_file",
"HashInDbError"]

8
src/pytestpavement/helper/exceptions.py Normal file
View File

@ -0,0 +1,8 @@
# define Python user-defined exceptions
class HashInDbError(Exception):
def __init__(self, value):
self.parameter = value
def __str__(self):
return repr(self.parameter)

67
src/pytestpavement/labtests/base.py
View File

@ -7,6 +7,7 @@ import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import scipy.fft as sfft
from pytestpavement.analysis import cosfunc
from pytestpavement.helper.filehasher import calc_hash_of_file
@ -284,15 +285,23 @@ class DataSineLoad():
idx_diff = np.diff(d.index)
dt_mean = idx_diff.mean()
gaps = idx_diff > (2 * dt_mean)
gaps = idx_diff > (4 * dt_mean)
has_gaps = any(gaps)
if has_gaps == False:
data_list.append(d)
else:
#FIX: GAP FINDING
data_list.append(d)
"""
print('has gaps')
print(gaps)
idx_gaps = (np.where(gaps)[0] - 1)[0]
print(idx_gaps)
data_list.append(d.iloc[0:idx_gaps])
"""
#add self.
if len(data_list) == 0:
@ -305,17 +314,61 @@ class DataSineLoad():
#break
def _select_data(self):
""" select N load cycles from original data """
"""
select N load cycles from original data
(a): Based on window of TP-Asphalt
(b) last N cycles
"""
def sel_df(df, num=5):
N = df['N'].unique()
freq = float(df['f'].unique()[0])
if len(N) > num:
df_sel = df[(df['N'] >= N[-num - 1]) & (df['N'] <= N[-2])]
return df_sel
# define cycles to select
if freq == 10.0:
Nfrom = 98
Nto = 103
elif freq == 5.0:
Nfrom = 93
Nto = 97
elif freq == 3.0:
Nfrom = 43
Nto = 47
elif freq == 1.0:
Nfrom = 13
Nto = 17
elif freq == 0.3:
Nfrom = 8
Nto = 12
elif freq == 0.1:
Nfrom = 3
Nto = 7
else:
ValueError(
'Number of load cycles smaller than selectect values')
Nfrom = None
Nto = None
if Nfrom != None:
if len(N) > Nto - Nfrom:
df_sel = df[(df['N'] >= Nfrom) & (df['N'] <= Nto)]
if self.debug:
print('select N', Nfrom, Nto, freq)
print(df_sel.head())
print(df_sel.tail())
return df_sel
else:
ValueError(
'Number of load cycles smaller than selectect values')
else:
if len(N) > num:
df_sel = df[(df['N'] >= N[-num - 1]) & (df['N'] <= N[-2])]
return df_sel
else:
ValueError(
'Number of load cycles smaller than selectect values')
if not isinstance(self.data, list):
if self.number_of_load_cycles_for_analysis > 1:

75
src/pytestpavement/labtests/citt.py
View File

@ -4,12 +4,16 @@ from csv import reader
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from bson import ObjectId
from pytestpavement.analysis.regression import fit_cos, fit_cos_eval
from pytestpavement.functions.citt import calc_nu
#from pytestpavement.helper.exceptions import HashInDbError
from pytestpavement.io.geosys import read_geosys
from pytestpavement.labtests.base import DataSineLoad
from pytestpavement.models.citt import CyclicIndirectTensileTest
from pytestpavement.models.citt import (CITTSiffnessResults,
CyclicIndirectTensileTest)
from pytestpavement.models.data import CITTSiffness
def check_timeindex(t, cycles: int, frequency: float, error: float = 10.0):
@ -55,6 +59,7 @@ class CittTest(DataSineLoad):
self.frequency = frequency
self.sigma = sigma
self.meta = meta
self.debug = debug
# process file
self._run()
@ -181,6 +186,44 @@ class CittTest(DataSineLoad):
self.fit = self.fit.reset_index(
drop=True).set_index('idx_data').sort_index()
def save(self, org_id: ObjectId, project_id: ObjectId, material_id: ObjectId, user_id: ObjectId = None, meta: dict = {}, wp_id=None):
for idx_fit, fit in self.fit.iterrows():
data = self.data[idx_fit]
meta['filehash'] = self.filehash
meta['org_id'] = org_id
meta['project_id'] = project_id
meta['material'] = material_id
meta['user_id'] = user_id
#check if result in db
#n = CITTSiffness.objects(**meta).count()
#print(n)
# write data
data_dict = fit.to_dict()
data_dict.update(meta)
f = CITTSiffnessResults(**data_dict).save()
data_out = dict(
time = data.index,
F= list(data['F']),
N = list(data['N']),
s_hor_1 = list(data['s_hor_1']),
s_hor_2 = list(data['s_hor_2']),
s_hor_sum = list(data['s_hor_sum']),
s_piston = list(data['s_piston']),
)
g = CITTSiffness(
result=f.id,
**data_out
).save()
class CittTestTUDGeosys(CittTest):
@ -284,7 +327,7 @@ class CittTestTUDTira(CittTest):
'time', 'T', 'f', 'N', 'F', 's_hor_sum', 's_hor_1', 's_hor_2'
]
# Header names after standardization; check if exists
self.val_header_names = ['speciment_diameter']
self.val_header_names = ['speciment_diameter', 'speciment_diameter']
self.columns_analyse = ['F', 's_hor_sum', 's_hor_1', 's_hor_2']
@ -297,7 +340,7 @@ class CittTestTUDTira(CittTest):
def _define_units(self):
self.unit_s = 1. #ym in mm
self.unit_F = 1 #N
self.unit_F = -1.0 #N
self.unit_t = 1. #s
def _read_data(self, skiprows=28, hasunits=True):
@ -308,7 +351,6 @@ class CittTestTUDTira(CittTest):
meta = {}
splitsign = ':;'
with open(self.file, 'r', encoding=self.encoding) as f:
count = 0
@ -349,7 +391,7 @@ class CittTestTUDTira(CittTest):
decimal=',',
sep=';')
data = data.iloc[1:]
data = data.iloc[2:]
data.columns = head
@ -414,13 +456,26 @@ class CittTestTUDTira(CittTest):
self.data.columns = colnames
def _calc_missiong_values(self):
def _calc_missiong_values(self):
cols = self.data.columns
cols = self.data.columns
if not 's_hor_sum' in cols:
self.data['s_hor_sum'] = self.data[['s_hor_1',
's_hor_2']].sum(axis=1)
if not 'f' in cols:
self.data['f'] = float(
self.meta['Sollfrequnz'].split(';')[0].strip().replace(
',', '.'))
if not 's_hor_sum' in cols:
self.data['s_hor_sum'] = self.data[['s_hor_1',
's_hor_2']].sum(axis=1)
self.sigma = float(
self.meta['Oberspannung'].split(';')[0].strip().replace(',', '.'))
self.temperature = float(
self.meta['Solltemperatur'].split(';')[0].strip().replace(
',', '.'))
self.frequency = float(
self.meta['Sollfrequnz'].split(';')[0].strip().replace(',', '.'))
class CittTestBASt(CittTest):

73
src/pytestpavement/labtests/sheartest.py
View File

@ -1,13 +1,17 @@
import os
import tempfile
import lmfit as lm
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import py7zr
import scipy.fft as sfft
import seaborn as sns
from pytestpavement.analysis.regression import fit_cos, fit_cos_eval
from pytestpavement.io.geosys import read_geosys
from pytestpavement.io.minio import MinioClient
from pytestpavement.labtests.base import DataSineLoad
from pytestpavement.models.data import DataSheartest
from pytestpavement.models.sheartest import DynamicShearTestExtension
@ -22,13 +26,17 @@ class ShearTest(DataSineLoad):
fname: str,
debug: bool = False,
gap_width: float = 1.0,
roundtemperature: bool = True):
roundtemperature: bool = True,
archive_file=False,
s3_params: dict = {}):
#set parameter
self.gap_width = gap_width
self.debug = debug
self.file = fname
self.roundtemperature = roundtemperature
self.archive_file = archive_file
self.s3_params = s3_params
# process file
self._run()
@ -159,6 +167,7 @@ class ShearTestExtension(ShearTest):
f=fit['f'],
sigma_normal=fit['sigma_normal'],
T=fit['T'],
extension=fit['extension'],
material1=material1,
material2=material2,
@ -167,17 +176,6 @@ class ShearTestExtension(ShearTest):
).count()
if n > 0: continue
#save data
rdata = DataSheartest(
time=data.index.values,
F=data['F'].values,
N=data['N'].values,
s_vert_1=data['s_vert_1'].values,
s_vert_2=data['s_vert_2'].values,
s_vert_sum=data['s_vert_sum'].values,
s_piston=data['s_piston'].values,
).save()
# save fit
values = {}
@ -203,15 +201,64 @@ class ShearTestExtension(ShearTest):
material2=material2,
bounding=bounding,
#results
data=rdata,
stiffness=fit['G'],
#
**values).save()
#save raw data
rdata = DataSheartest(
result_id=r.id,
time=data.index.values,
F=data['F'].values,
N=data['N'].values,
s_vert_1=data['s_vert_1'].values,
s_vert_2=data['s_vert_2'].values,
s_vert_sum=data['s_vert_sum'].values,
s_piston=data['s_piston'].values,
).save()
except:
print('error saving data')
raise
rdata.delete()
if self.archive_file:
mclient = MinioClient(self.s3_params['S3_URL'],
self.s3_params['S3_ACCESS_KEY'],
self.s3_params['S3_SECRET_KEY'])
bucket = str(meta['org_id'])
mclient.create_bucket(bucket)
extension = os.path.splitext(self.file)[-1]
ofilename = self.filehash + extension
outpath = 'sheartest'
metadata_s3 = {
'project_id': str(meta['project_id']),·
'user_id': str(meta['user_id']),
'filename': os.path.split(self.file)[-1],
'speciment': meta['speciment_name']
}
#compress data to tmpfolder
with tempfile.TemporaryDirectory() as tmpdirname:
ofilename_compressed = ofilename + '.7z'
compressed_file = os.path.join(tmpdirname,
ofilename_compressed)
with py7zr.SevenZipFile(compressed_file, 'w') as archive:
archive.writeall(self.file)
mclient.upload_file(bucket,
compressed_file,
ofilename_compressed,
outpath=outpath,
content_type="application/raw",
metadata=metadata_s3)
def _set_parameter(self):
self.split_data_based_on_parameter = [

1
src/pytestpavement/models/__init__.py
View File

@ -2,4 +2,5 @@ from .citt import *
from .material import *
from .project import *
from .sheartest import *
from .usermanagement import *
from .workpackage import *

29
src/pytestpavement/models/citt.py
View File

@ -2,8 +2,10 @@ import datetime
from mongoengine import *
from .data import CITTSiffness
from .material import Material
from .project import Project
from .usermanagement import Organisation, User
from .workpackage import Workpackage
class CyclicIndirectTensileTest(Document):
@ -15,15 +17,22 @@ class CyclicIndirectTensileTest(Document):
standard = StringField(default='TP Asphalt Teil 24')
lab = StringField(default='TU Dresden', required=True)
auditor = StringField(default=None)
org_id = LazyReferenceField(Organisation, required=True)
project_id = LazyReferenceField(Project, required=True)
workpackage_id = LazyReferenceField(Workpackage, required=False)
user_id = LazyReferenceField(User,
required=True,
reverse_delete_rule=DO_NOTHING)
material = LazyReferenceField(Material, required=True)
tags = ListField(StringField())
machine = StringField(default=None)
filehash = StringField(required=True)
project = StringField(required=True)
workpackage = StringField()
material = ReferenceField(Material, required=True)
speciment_name = StringField()
meta = {
'allow_inheritance': True,
@ -35,7 +44,7 @@ class CyclicIndirectTensileTest(Document):
}
class CITTSiffness(CyclicIndirectTensileTest):
class CITTSiffnessResults(CyclicIndirectTensileTest):
#metadata
f_set = FloatField()
@ -45,10 +54,6 @@ class CITTSiffness(CyclicIndirectTensileTest):
N_from = IntField()
N_to = IntField()
data = LazyReferenceField(CITTSiffness,
required=True,
reverse_delete_rule=CASCADE)
#results
stiffness = FloatField()
nu = FloatField()

12
src/pytestpavement/models/data.py
View File

@ -2,6 +2,9 @@ import datetime
from mongoengine import *
from .citt import CyclicIndirectTensileTest
from .sheartest import DynamicShearTest
class RawData(Document):
@ -22,6 +25,11 @@ class RawData(Document):
class DataSheartest(RawData):
#results
result_id = LazyReferenceField(DynamicShearTest,
required=True,
reverse_delete_rule=CASCADE)
# data
time = ListField(FloatField())
F = ListField(FloatField())
@ -34,6 +42,10 @@ class DataSheartest(RawData):
class CITTSiffness(RawData):
result = LazyReferenceField(CyclicIndirectTensileTest,
required=True,
reverse_delete_rule=CASCADE)
# data
time = ListField(FloatField())
F = ListField(FloatField())

36
src/pytestpavement/models/material.py
View File

@ -1,7 +1,11 @@
import datetime
from re import T
from mongoengine import *
from .project import Project
from .usermanagement import Organisation, User
class Material(Document):
@ -10,8 +14,20 @@ class Material(Document):
"step": "60"
}})
org_id = LazyReferenceField(Organisation,
required=True,
reverse_delete_rule=CASCADE)
project_id = ListField(LazyReferenceField(Project,
required=False,
reverse_delete_rule=CASCADE),
required=True)
user_id = LazyReferenceField(User,
required=False,
reverse_delete_rule=DO_NOTHING)
tags = ListField(StringField())
project = StringField(required=False)
norm = StringField(required=True, default='TP Asphalt Teil 24')
meta = {
@ -20,7 +36,11 @@ class Material(Document):
'index_background': True,
'index_cls': False,
'auto_create_index': True,
'collection': 'materials'
'collection': 'materials',
'indexes': [
[("material", 1)],
[("name", 1)],
]
}
@ -42,6 +62,14 @@ class Bitumen(Material):
young_modulus = DictField()
class Bitumenemulsion(Material):
name = StringField()
material = StringField()
young_modulus = DictField()
class Epoxy(Material):
name = StringField()
material = StringField()
@ -49,6 +77,10 @@ class Epoxy(Material):
young_modulus = DictField()
class Kompaktasphalt(Material):
name = StringField()
class Dummy(Material):
name = StringField()

19
src/pytestpavement/models/project.py
View File

@ -2,18 +2,29 @@ import datetime
from mongoengine import *
from .usermanagement import Organisation, User
class Project(Document):
name = StringField(required=True)
name_short = StringField(required=False)
project_id = StringField(required=True)
client = StringField(required=False)
date = DateTimeField(default=datetime.datetime.now,
wtf_options={"render_kw": {
"step": "60"
}})
org_id = LazyReferenceField(Organisation,
required=True,
reverse_delete_rule=CASCADE)
user_id = LazyReferenceField(User,
required=False,
reverse_delete_rule=DO_NOTHING)
tags = ListField(StringField())
meta = {
@ -22,6 +33,10 @@ class Project(Document):
'index_background': True,
'index_cls': False,
'auto_create_index': True,
'collection': 'projects'
'collection': 'projects',
'indexes': [
[("name_short", 1)],
[("name", 1)],
[("project_id", 1)],
]
}

58
src/pytestpavement/models/sheartest.py
View File

@ -2,9 +2,9 @@ import datetime
from mongoengine import *
from .data import DataSheartest
from .material import Material
from .project import Project
from .usermanagement import Organisation, User
from .workpackage import Workpackage
@ -15,19 +15,13 @@ class DynamicShearTest(Document):
"step": "60"
}})
tags = ListField(StringField())
org_id = LazyReferenceField(Organisation, required=True)
project_id = LazyReferenceField(Project, required=True)
workpackage_id = LazyReferenceField(Workpackage, required=False)
standard = StringField(default='TP Asphalt Teil 24')
lab = StringField(default='TU Dresden', required=True)
auditor = StringField(default=None)
machine = StringField(default=None)
filehash = StringField(required=True)
speciment_name = StringField()
project = LazyReferenceField(Project, required=True)
workpackage = LazyReferenceField(Workpackage, required=True)
user_id = LazyReferenceField(User,
required=True,
reverse_delete_rule=DO_NOTHING)
material1 = LazyReferenceField(Material, required=True)
material2 = LazyReferenceField(Material, required=True)
@ -35,13 +29,36 @@ class DynamicShearTest(Document):
gap_width = FloatField(default=1.0)
tags = ListField(StringField())
standard = StringField(default='TP Asphalt Teil 24')
machine = StringField(default=None)
filehash = StringField(required=True)
speciment_name = StringField()
meta = {
'allow_inheritance': True,
'allow_inheritance':
True,
'index_opts': {},
'index_background': True,
'index_cls': False,
'auto_create_index': True,
'collection': 'sheartest',
'index_background':
True,
'index_cls':
False,
'auto_create_index':
True,
'collection':
'sheartest',
'indexes': [
[("lab", 1)],
[("speciment_name", 1)],
[("project", 1)],
[("bruch", 1)],
[("lab", 1), ("project", 1)],
[("lab", 1), ("project", 1), ("workpackage", 1)],
[("lab", 1), ("project", 1), ("bounding", 1)],
]
}
@ -53,11 +70,6 @@ class DynamicShearTestExtension(DynamicShearTest):
T = FloatField(required=True)
extension = FloatField(required=True)
#results
data = LazyReferenceField(DataSheartest,
required=True,
reverse_delete_rule=CASCADE)
stiffness = FloatField(required=True)
bruch = BooleanField(required=True)
#fit parameter

48
src/pytestpavement/models/usermanagement.py Normal file
View File

@ -0,0 +1,48 @@
import datetime
from mongoengine import *
class Organisation(Document):
name = StringField(required=True)
name_short = StringField(required=True)
date = DateTimeField(default=datetime.datetime.now,
wtf_options={"render_kw": {
"step": "60"
}})
meta = {
'allow_inheritance': True,
'index_opts': {},
'index_background': True,
'index_cls': False,
'auto_create_index': True,
'collection': 'organisation'
}
class User(Document):
active = BooleanField(required=True, default=True)
org_id = LazyReferenceField(Organisation,
required=True,
reverse_delete_rule=CASCADE)
date_added = DateTimeField(default=datetime.datetime.now,
wtf_options={"render_kw": {
"step": "60"
}})
name = StringField(required=True)
meta = {
'allow_inheritance': True,
'index_opts': {},
'index_background': True,
'index_cls': False,
'auto_create_index': True,
'collection': 'user'
}

13
src/pytestpavement/models/workpackage.py
View File

@ -3,6 +3,7 @@ import datetime
from mongoengine import *
from .project import Project
from .usermanagement import User
class Workpackage(Document):
@ -11,12 +12,16 @@ class Workpackage(Document):
name_short = StringField(required=False)
wp_id = StringField(required=True)
project = LazyReferenceField(Project, required=True)
project_id = LazyReferenceField(Project, required=True)
user_id = LazyReferenceField(User,
required=False,
reverse_delete_rule=DO_NOTHING)
date = DateTimeField(default=datetime.datetime.now,
wtf_options={"render_kw": {
"step": "60"
}})
wtf_options={"render_kw": {
"step": "60"
}})
meta = {
'allow_inheritance': True,