@conference {2019:icpe:benchflow, title = {Behavior-driven Load Testing Using Contextual Knowledge {\textemdash} Approach and Experiences}, booktitle = {10th ACM/SPEC International Conference on Performance Engineering (ICPE 2019)}, year = {2019}, month = {April}, pages = {265-272}, publisher = {ACM/SPEC}, organization = {ACM/SPEC}, address = {Mumbai, India}, abstract = {Load testing is widely considered a meaningful technique for performance quality assurance. However, empirical studies reveal that in practice, load testing is not applied systematically, due to the sound expert knowledge required to specify, implement, and execute load tests. Our Behavior-driven Load Testing (BDLT) approach eases load test specification and execution for users with no or little expert knowledge. It allows a user to describe a load test in a template based on natural language and to rely on an automated framework to execute the test. Utilizing the system{\textquoteright}s contextual knowledge such as workload-influencing events, the framework automatically determines the workload and test configuration. We investigated the applicability of our approach in an industrial case study, where we were able to express four load test concerns using BDLT and received positive feedback from our industrial partner. They understood the BDLT definitions well and proposed further applications, such as the usage for software quality acceptance criteria.}, keywords = {BenchFlow, Performance Testing}, doi = {10.1145/3297663.3309674}, author = {Henning Schulz and Du{\v s}an Okanovi{\'c} and Andr{\'e} van Hoorn and Vincenzo Ferme and Cesare Pautasso} } @conference {benchflow:2016:caise, title = {Micro-Benchmarking BPMN 2.0 Workflow Management Systems with Workflow Patterns}, booktitle = {Proc. of the 28th International Conference on Advanced Information Systems Engineering (CAISE)}, year = {2016}, month = {June}, pages = {67--82}, publisher = {Springer}, organization = {Springer}, address = {Ljubljana, Slovenia}, abstract = {Although Workflow Management Systems (WfMSs) are a key component in workflow technology, research work for assessing and comparing their performance is limited. This work proposes the first micro- benchmark for WfMSs that can execute BPMN 2.0 workflows. To this end, we focus on studying the performance impact of well-known workflow patterns expressed in BPMN 2.0 with respect to three open source WfMSs (i.e., Activiti, jBPM and Camunda). We executed all the experiments under a reliable environment and produced a set of meaningful metrics. This paper contributes to the area of workflow technology by defining building blocks for more complex BPMN 2.0 WfMS benchmarks. The results have shown bottlenecks on architectural design decisions, resource utilization, and limits on the load a WfMS can sustain, especially for the cases of complex and parallel structures. Experiments on a mix of workflow patterns indicated that there are no unexpected performance side effects when executing different workflow patterns concurrently, although the duration of the individual workflows that comprised the mix was increased.}, keywords = {BenchFlow, benchmarking, BPMN, Microbenchmark, workflow engine, Workflow Management Systems, workflow patterns}, doi = {10.1007/978-3-319-39696-5_5}, url = {http://www2.informatik.uni-stuttgart.de/cgi-bin/NCSTRL/NCSTRL_view.pl?id=INPROC-2016-05\&engl=0}, author = {Marigianna Skouradaki and Vincenzo Ferme and Cesare Pautasso and Frank Leymann and Andr{\'e} van Hoorn} }