@phdthesis {2021:phd:vincenzo-ferme,
	title = {Declarative Performance Testing Automation: Automating Performance Testing for the DevOps Era},
	year = {2021},
	month = {January},
	school = {USI},
	type = {PhD},
	address = {Lugano},
	abstract = {Recent trends in industry show increasing adoption of Development and Operations (DevOps) practices. Reasons for increasing DevOps adoption are the focus on the creation of cross-functional teams, and the ability to release high-quality software at a fast pace. Alongside the adoption of DevOps, performance testing continues to evolve to meet the growing demands of the modern enterprise and its need for automation. As DevOps adoption continues and self-service environment provisioning becomes commonplace in Information Technology (IT) departments, more developers will be working on executing performance tests, to ensure the quality of released services satisfies users{\textquoteright} expectations while constraining the resources needed to do so. 

Modeling and automated execution of performance tests are time-consuming and difficult activities, requiring expert knowledge, complex infrastructure, and a rigorous process to guarantee the quality of collected performance data and the obtained results. Currently available performance testing approaches are not well integrated with DevOps practices and tools and are often focusing only on specific needs of performance testing modeling and automation. 

A recent survey by the Standard Performance Evaluation Corporation (SPEC) Research Group (RG) on DevOps reported the need for a new paradigm for performance activities to be successfully integrated with DevOps practices and tools, such as the one proposed by Declarative Performance Engineering (DPE). Previous studies reported successful applications of DPE to DevOps contexts, due to the opportunity to model the performance testing domain knowledge as a first-class citizen and its ability to offer different levels of abstraction to different people relying on it. 

In this dissertation, we introduce a "Declarative Approach for Performance Tests Execution Automation" enabling the continuous and automated execution of performance tests alongside the Continuous Software Development Lifecycle (CSDL), an integral part of DevOps practices. We contribute an automation-oriented catalog of performance test types and goals and a description of how they fit in different moments of the CSDL, a declarative Domain Specific Language (DSL) enabling the declarative specification of performance tests and their automated orchestration processes alongside the CSDL, and a framework for end-to-end automated performance testing of RESTful (RESTful) Web services and Business Process Model and Notation 2.0 (BPMN 2.0) Workflow Management Systems (WfMSs) relying on the contributed DSL. We evaluate the proposed DSL by conducting an expert review targeting its overall expressiveness and suitability for the target users, perceived usability and effort, and reusability of specified tests. We also perform a summative evaluation of the DSL{\textquoteright}s usability in terms of learnability, and reusability of test specifications. The surveys confirm the proposed approach is valid for the aims it has been built for, and it is considered on average good for all the evaluated usability dimensions. We evaluate the implemented framework by performing iterative reviews of the different versions of the framework, and a comparative evaluation of the proposed framework{\textquoteright}s features compared to state-of-the-art available solutions. The iterative reviews led to many improvements due to the received constructive feedback, while the comparative evaluation showed no other solutions similar to the proposed one are available in the literature. We assess the overall contributed solution by executing a large number of case studies, by collaborating with other researchers in extending both the DSL and the framework.},
	keywords = {Performance Testing},
	url = {https://doc.rero.ch/record/330229?ln=en},
	author = {Vincenzo Ferme}
}
@conference {2019:icpe:benchflow,
	title = {Behavior-driven Load Testing Using Contextual Knowledge {\textemdash} Approach and Experiences},
	booktitle = {10th ACM/SPEC International Conference on Performance Engineering (ICPE 2019)},
	year = {2019},
	month = {April},
	pages = {265-272},
	publisher = {ACM/SPEC},
	organization = {ACM/SPEC},
	address = {Mumbai, India},
	abstract = {Load testing is widely considered a meaningful technique for performance quality assurance. However, empirical studies reveal that in practice, load testing is not applied systematically, due to the sound expert knowledge required to specify, implement, and execute load tests. Our Behavior-driven Load Testing (BDLT) approach eases load test specification and execution for users with no or little expert knowledge. It allows a user to describe a load test in a template based on
natural language and to rely on an automated framework to execute the test. Utilizing the system{\textquoteright}s contextual knowledge such as workload-influencing events, the framework automatically determines the workload and test configuration. We investigated the applicability of our approach in an industrial case study, where we were able to express four load test concerns using BDLT and received positive feedback from our industrial partner. They understood the BDLT definitions well and proposed further applications, such as the usage for software quality acceptance criteria.},
	keywords = {BenchFlow, Performance Testing},
	doi = {10.1145/3297663.3309674},
	author = {Henning Schulz and Du{\v s}an Okanovi{\'c} and Andr{\'e} van Hoorn and Vincenzo Ferme and Cesare Pautasso}
}
@conference {2018:benchflow:coopis,
	title = {Evaluating Multi-Tenant Live Migrations Effects on Performance},
	booktitle = {26th International Conference on Cooperative Information Systems (CoopIS)},
	year = {2018},
	month = {October},
	address = {Valletta, Malta},
	abstract = {Multitenancy is an important feature for all Everything as a Service providers like Business Process Management as a Service. It allows to reduce the cost of the infrastructure since multiple tenants share the same service instances. However, tenants have dynamic workloads. The resource they share may not be sufficient at some point in time. It may require Cloud resource (re-)configurations to ensure a given Quality of Service. Tenants should be migrated without stopping the service from a configuration to another to meet their needs while minimizing operational costs on the provider side. Live migrations reveal many challenges: service interruption must be minimized and the impact on co-tenants should be minimal. In this paper, we investigate live tenants migrations duration and its effects on the migrated tenants as well as the co-located ones. To do so, we propose a generic approach to measure these effects for multi-tenant Software as a Service. Further, we propose a testing framework to simulate workloads, and observe the impact of live migrations on Business Process Management Systems. The experimental results highlight the efficiency of our approach and show that migration time depends on the size of data that have to be transferred and that the effects on co-located tenants should not be neglected. },
	keywords = {BenchFlow, Multi-tenant, Performance Testing, workflow engine},
	author = {Guillaume Rosinosky and Chahrazed Labba and Vincenzo Ferme and Samir Youcef and Fran{\c c}ois Charoy and Cesare Pautasso}
}
@proceedings {2017:benchflow:bpmds,
	title = {Performance Comparison Between BPMN 2.0 Workflow Management Systems Versions},
	year = {2017},
	month = {June},
	publisher = {Springer},
	address = {Essen, Germany},
	abstract = {Software has become a rapidly evolving artifact and Workflow Management Systems (WfMSs) are not an exception. WfMSs{\textquoteright} changes may impact key performance indicators or resource consumption levels may change among different versions. Thus, users considering a WfMS upgrade need to evaluate the extent of such changes for frequently issued workload.  Deriving such information requires running performance experiments with appropriate workloads. 
In this paper, we propose a novel method for deriving a structurally representative workload from a given business process collection, which we later use to evaluate the performance and resource consumption over four versions of two open-source WfMSs, for different numbers of simulated users. In our case study scenario the results reveal relevant variations in the WfMSs{\textquoteright} performance and resource consumption, indicating a decrease in performance for newer versions.},
	keywords = {BenchFlow, BPMN, Performance Regression, Performance Testing, workflow engine, Workflow Management Systems},
	author = {Vincenzo Ferme and Marigianna Skouradaki and Ana Ivanchikj and Cesare Pautasso and Frank Leymann}
}
