Publications

@article{10.1145/3701764,
author = {Peintner, Andreas and Mohammadi, Amir Reza and Zangerle, Eva},
title = {Efficient Session-based Recommendation with Contrastive Graph-based Shortest Path Search},
year = {2024},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3701764},
doi = {10.1145/3701764},
abstract = {Session-based recommendation aims to predict the next item based on a set of anonymous sessions. Capturing user intent from a short interaction sequence imposes a variety of challenges since no user profiles are available and interaction data is naturally sparse. Recent approaches relying on graph neural networks (GNNs) for session-based recommendation use global item relations to explore collaborative information from different sessions. These methods capture the topological structure of the graph and rely on multi-hop information aggregation in GNNs to exchange information along edges. Consequently, graph-based models suffer from noisy item relations in the training data and introduce high complexity for large item catalogs. We propose to explicitly model the multi-hop information aggregation mechanism over multiple layers via shortest-path edges based on knowledge from the sequential recommendation domain. Our approach does not require multiple layers to exchange information and ignores unreliable item-item relations. Furthermore, to address inherent data sparsity, we are the first to apply supervised contrastive learning by mining data-driven positive and hard negative item samples from the training data. Extensive experiments on four different datasets show that the proposed approach outperforms almost all of the state-of-the-art methods.},
note = {Just Accepted},
journal = {ACM Trans. Recomm. Syst.},
month = oct,
keywords = {Recommender Systems, Session-based Recommendation, Graph Neural Network, Supervised Contrastive Learning}
}

@Article{bauer:dagstuhl:2024,
  author =	{Bauer, Christine and Said, Alan and Zangerle, Eva},
  title =	{{Evaluation Perspectives of Recommender Systems: Driving Research and Education (Dagstuhl Seminar 24211)}},
  pages =	{58--172},
  journal =	{Dagstuhl Reports},
  ISSN =	{2192-5283},
  year =	{2024},
  volume =	{14},
  number =	{5},
  editor =	{Bauer, Christine and Said, Alan and Zangerle, Eva},
  publisher =	{Schloss Dagstuhl -- Leibniz-Zentrum f{\"u}r Informatik},
  address =	{Dagstuhl, Germany},
  URL =		{https://drops.dagstuhl.de/entities/document/10.4230/DagRep.14.5.58},
  URN =		{urn:nbn:de:0030-drops-222655},
  doi =		{10.4230/DagRep.14.5.58},
  annote =	{Keywords: Recommender Systems, Evaluation, Information Retrieval, User Interaction, Intelligent Systems}
}

@inproceedings{10.1145/3640457.3687101,
author = {Said, Alan and Bauer, Christine and Zangerle, Eva},
title = {Reflections on Recommender Systems: Past, Present, and Future (INTROSPECTIVES)},
year = {2024},
isbn = {9798400705052},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3640457.3687101},
doi = {10.1145/3640457.3687101},
abstract = {With the RecSys conference now turning 18 years old, the recommender systems (RS) discipline ventures into adulthood. This workshop serves as a platform for introspection, examining the evolution of RS from its origins in CHI to its current state heavily influenced by and focusing on machine learning. The INTROSPECTIVES workshop aims to foster discussions on the past, present, and future of the RS discipline, inviting the community to reflect on key questions such as the maturation of RS, shifts in research focus, and the impact and success of RS in practice. Topics include the changing landscape of RS problems, the evolving role of RS in addressing choice overload to the current motivations driving RS adoption.},
booktitle = {Proceedings of the 18th ACM Conference on Recommender Systems},
pages = {1237–1238},
numpages = {2},
keywords = {information retrieval, introspection, personalized systems, recommender systems, reflection},
location = {Bari, Italy},
series = {RecSys '24}
}

@InProceedings{10.1007/978-3-031-56072-9_1,
author="Bevendorff, Janek
and Casals, Xavier Bonet
and Chulvi, Berta
and Dementieva, Daryna
and Elnagar, Ashaf
and Freitag, Dayne
and Fr{\"o}be, Maik
and Koren{\v{c}}i{\'{c}}, Damir
and Mayerl, Maximilian
and Mukherjee, Animesh
and Panchenko, Alexander
and Potthast, Martin
and Rangel, Francisco
and Rosso, Paolo
and Smirnova, Alisa
and Stamatatos, Efstathios
and Stein, Benno
and Taul{\'e}, Mariona
and Ustalov, Dmitry
and Wiegmann, Matti
and Zangerle, Eva",
editor="Goharian, Nazli
and Tonellotto, Nicola
and He, Yulan
and Lipani, Aldo
and McDonald, Graham
and Macdonald, Craig
and Ounis, Iadh",
title="Overview of PAN 2024: Multi-author Writing Style Analysis, Multilingual Text Detoxification, Oppositional Thinking Analysis, and Generative AI Authorship Verification",
booktitle="Advances in Information Retrieval",
year="2024",
publisher="Springer Nature Switzerland",
address="Cham",
pages="3--10",
abstract="The paper gives a brief overview of the four shared tasks organized at the PAN 2024 lab on digital text forensics and stylometry to be hosted at CLEF 2024. The goal of the PAN lab is to advance the state-of-the-art in text forensics and stylometry through an objective evaluation of new and established methods on new benchmark datasets. Our four tasks are: (1) multi-author writing style analysis, which we continue from 2023 in a more difficult version, (2) multilingual text detoxification, a new task that aims to translate and re-formulate text in a non-toxic way, (3) oppositional thinking analysis, a new task that aims to discriminate critical thinking from conspiracy narratives and identify their core actors, and (4) generative AI authorship verification, which formulates the detection of AI-generated text as an authorship problem, one of PAN's core tasks. As with the previous editions, PAN invites software submissions as easy-to-reproduce docker containers; more than 400 pieces of software have been submitted from PAN'12 through PAN'23 combined, with all recent evaluations running on the TIRA experimentation platform [8].",
isbn="978-3-031-56072-9"
}

@inproceedings{zangerle:pan:2024,
  author       = {Eva Zangerle and
                  Maximilian Mayerl and
                  Martin Potthast and
                  Benno Stein},
  title        = {Overview of the Multi-Author Writing Style Analysis Task at {PAN}
                  2024},
  booktitle    = {Working Notes of the Conference and Labs of the Evaluation Forum {(CLEF}
                  2024), Grenoble, France, 9-12 September, 2024},
  series       = {{CEUR} Workshop Proceedings},
  volume       = {3740},
  pages        = {2424--2431},
  publisher    = {CEUR-WS.org},
  year         = {2024},
  url          = {https://ceur-ws.org/Vol-3740},
}

@article{jacobsen:musiscaescientiae:2024,
	title        = {Assessing aesthetic music-evoked emotions in a minute or less: A comparison of the GEMS-45 and the GEMS-9},
	author       = {Peer-Ole Jacobsen and Hannah Strauss and Julia Vigl and Eva Zangerle and Marcel Zentner},
	year         = {0},
	journal      = {Musicae Scientiae},
	pages        = 10298649241256252,
	doi          = {10.1177/10298649241256252},
	eprint       = {https://doi.org/10.1177/10298649241256252}
}

@InProceedings{murauer2022dtgrams,
  author       = {Amir Reza Mohammadi and
                  Amir{-}Hossein Karimi and
                  Mahdi Bohlouli and
                  Eva Zangerle and
                  G{\"{u}}nther Specht},
  booktitle = {Proceedings of the 34nd GI-Workshop Grundlagen von Datenbanksysteme (GvDB'23) },
  title     = {HPT4Rec: AutoML-based Hyperparameter Self-Tuning Framework for Session-based
                  Recommender Systems},
  year      = {2024},
}

@article{strauss:2024:behaviorresearchmethods,
	title        = {The Emotion-to-Music Mapping Atlas (EMMA): A systematically organized online database of emotionally evocative music excerpts},
	author       = {Strauss, Hannah and Vigl, Julia and Jacobsen, Peer-Ole and Bayer, Martin and Talamini, Francesca and Vigl, Wolfgang and Zangerle, Eva and Zentner, Marcel},
	year         = 2024,
	month        = {jun},
	journal      = {Behavior Research Methods},
	publisher    = {Springer},
	pages        = {3560--3577},
        issue_date   = {June 2024},
	volume	     = 56,
        number       = 4,
	doi          = {https://doi.org/10.3758/s13428-024-02336-0}
}

@inproceedings{10.1145/3627043.3659540,
author = {Moscati, Marta and Strau\ss{}, Hannah and Jacobsen, Peer-Ole and Peintner, Andreas and Zangerle, Eva and Zentner, Marcel and Schedl, Markus},
title = {Emotion-Based Music Recommendation from Quality Annotations and Large-Scale User-Generated Tags},
year = {2024},
isbn = {9798400704338},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3627043.3659540},
doi = {10.1145/3627043.3659540},
abstract = {Emotions constitute an important aspect when listening to music. While manual annotations from user studies grounded in psychological research on music and emotions provide a well-defined and fine-grained description of the emotions evoked when listening to a music track, user-generated tags provide an alternative view stemming from large-scale data. In this work, we examine the relationship between these two emotional characterizations of music and analyze their impact on the performance of emotion-based music recommender systems individually and jointly. Our analysis shows that (i) the agreement between the two characterizations, as measured with Cohen’s κ coefficient and Kendall rank correlation, is often low, (ii) Leveraging the emotion profile based on the intensity of evoked emotions from high-quality annotations leads to performances that are stable across different recommendation algorithms; (iii) Simultaneously leveraging the emotion profiles based on high-quality and large-scale annotations allows to provide recommendations that are less exposed to the low accuracy that algorithms might reach when leveraging one type of data, only.},
booktitle = {Proceedings of the 32nd ACM Conference on User Modeling, Adaptation and Personalization},
pages = {159–164},
numpages = {6},
keywords = {Annotation Study, Emotion-based Recommender Systems, Emotions, Music, Music Recommender Systems},
location = {Cagliari, Italy},
series = {UMAP '24}
}

@InProceedings{murauer2022dtgrams,
  author       = {Michael M. M{\"{u}}ller and
                  G{\"{u}}nther Specht and
                  Lukas Kleinheinz and
                  Janette F. Walde},
  booktitle = {Proceedings of the 35nd GI-Workshop Grundlagen von Datenbanksysteme (GvDB'24) },
  title        = {Bias Correction and Machine Learning in {AR(1)} Estimation: Bridging
                  Traditional and Modern Techniques},
  year      = {2024},
}

@proceedings{soto:iuiworkshops:2024,
  editor       = {Axel Soto and
                  Eva Zangerle},
  title        = {Joint Proceedings of the {ACM} {IUI} 2024 Workshops co-located with
                  the 29th Annual {ACM} Conference on Intelligent User Interfaces {(IUI}
                  2024), Greenville, South Carolina, USA, March 18, 2024},
  series       = {{CEUR} Workshop Proceedings},
  volume       = {3660},
  publisher    = {CEUR-WS.org},
  year         = {2024},
  url          = {https://ceur-ws.org/Vol-3660},
  urn          = {urn:nbn:de:0074-3660-6},
  timestamp    = {Fri, 12 Apr 2024 20:58:03 +0200},
  biburl       = {https://dblp.org/rec/conf/iui/2024w.bib},
  bibsource    = {dblp computer science bibliography, https://dblp.org}
}

@article{parada:srep:2024,
  title={Song lyrics have become simpler and more repetitive over the last five decades},
  author={Parada-Cabaleiro, Emilia and Mayerl, Maximilian and Brandl, Stefan and Skowron, Marcin and Schedl, Markus and Lex, Elisabeth and Zangerle, Eva},
  journal={Scientific Reports},
  volume={14},
  number={1},
  pages={5531},
  year={2024},
  publisher={Nature Publishing Group UK London},
  doi={10.1038/s41598-024-55742-x}
}

@article{10.1145/3629170,
author = {Bauer, Christine and Zangerle, Eva and Said, Alan},
title = {Exploring the Landscape of Recommender Systems Evaluation: Practices and Perspectives},
year = {2024},
issue_date = {March 2024},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
volume = {2},
number = {1},
url = {https://doi.org/10.1145/3629170},
doi = {10.1145/3629170},
abstract = {Recommender systems research and practice are fast-developing topics with growing adoption in a wide variety of information access scenarios. In this article, we present an overview of research specifically focused on the evaluation of recommender systems. We perform a systematic literature review, in which we analyze 57 papers spanning six years (2017–2022). Focusing on the processes surrounding evaluation, we dial in on the methods applied, the datasets utilized, and the metrics used. Our study shows that the predominant experiment type in research on the evaluation of recommender systems is offline experimentation and that online evaluations are primarily used in combination with other experimentation methods, e.g., an offline experiment. Furthermore, we find that only a few datasets (MovieLens, Amazon review dataset) are widely used, while many datasets are used in only a few papers each. We observe a similar scenario when analyzing the employed performance metrics—a few metrics are widely used (precision, normalized Discounted Cumulative Gain, and Recall), while many others are used in only a few papers. Overall, our review indicates that beyond-accuracy qualities are rarely assessed. Our analysis shows that the research community working on evaluation has focused on the development of evaluation in a rather narrow scope, with the majority of experiments focusing on a few metrics, datasets, and methods.},
journal = {ACM Trans. Recomm. Syst.},
month = {mar},
articleno = {11},
numpages = {31},
keywords = {Evaluation, survey, systematic literature review, recommender systems}
}

@inproceedings{DBLP:conf/ecir/BevendorffCCDEFFKMMPPRRSSSTUWZ24,
  author       = {Janek Bevendorff and
                  Xavier Bonet Casals and
                  Berta Chulvi and
                  Daryna Dementieva and
                  Ashaf Elnagar and
                  Dayne Freitag and
                  Maik Fr{\"{o}}be and
                  Damir Korencic and
                  Maximilian Mayerl and
                  Animesh Mukherjee and
                  Alexander Panchenko and
                  Martin Potthast and
                  Francisco Rangel and
                  Paolo Rosso and
                  Alisa Smirnova and
                  Efstathios Stamatatos and
                  Benno Stein and
                  Mariona Taul{\'{e}} and
                  Dmitry Ustalov and
                  Matti Wiegmann and
                  Eva Zangerle},
  editor       = {Nazli Goharian and
                  Nicola Tonellotto and
                  Yulan He and
                  Aldo Lipani and
                  Graham McDonald and
                  Craig Macdonald and
                  Iadh Ounis},
  title        = {Overview of {PAN} 2024: Multi-author Writing Style Analysis, Multilingual
                  Text Detoxification, Oppositional Thinking Analysis, and Generative
                  {AI} Authorship Verification - Extended Abstract},
  booktitle    = {Advances in Information Retrieval - 46th European Conference on Information
                  Retrieval, {ECIR} 2024, Glasgow, UK, March 24-28, 2024, Proceedings,
                  Part {VI}},
  series       = {Lecture Notes in Computer Science},
  volume       = {14613},
  pages        = {3--10},
  publisher    = {Springer},
  year         = {2024},
  doi          = {10.1007/978-3-031-56072-9\_1},
}

@article{10.1145/3648398,
author = {Bauer, Christine and Said, Alan and Zangerle, Eva},
title = {Introduction to the Special Issue on Perspectives on Recommender Systems Evaluation},
year = {2024},
issue_date = {March 2024},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
volume = {2},
number = {1},
url = {https://doi.org/10.1145/3648398},
doi = {10.1145/3648398},
abstract = {Evaluation plays a vital role in recommender systems—in research and practice—whether for confirming algorithmic concepts or assessing the operational validity of designs and applications. It may span the evaluation of early ideas and approaches up to elaborate implementations of systems integrated into everyday product settings; it may target a wide spectrum of different factors being evaluated. In this special issue, we explore recommender systems evaluation—theory and practice—while considering a diverse set of perspectives. These include recommender systems purposes, stakeholders, methodological approaches, and consequences. The collection of articles in this special issue offers insightful analyses of current recommender system evaluation practices, acknowledging their limitations, and setting out future research directions. As recommender systems evolve, the need for adequate evaluation methods and approaches increases. This special issue sheds light on areas undergoing development or requiring added attention from the research and practitioner communities in recommender systems. The compilation serves as a call to the recommender systems research community, motivating continued research and exploration of evaluation metrics, methods, and strategies.},
journal = {ACM Trans. Recomm. Syst.},
month = {mar},
articleno = {1},
numpages = {5},
keywords = {Evaluation, recommender systems, reproducibility, datasets, metrics}
}

@article{10.1145/3642979.3643000,
author = {Said, Alan and Zangerle, Eva and Bauer, Christine},
title = {Report on the 3rd Workshop on the Perspectives on the Evaluation of Recommender Systems (PERSPECTIVES 2023) at RecSys 2023},
year = {2024},
issue_date = {December 2023},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
volume = {57},
number = {2},
issn = {0163-5840},
url = {https://doi.org/10.1145/3642979.3643000},
doi = {10.1145/3642979.3643000},
journal = {SIGIR Forum},
month = {jan},
articleno = {18},
numpages = {4}
}

@inproceedings{10.1145/3640457.3691709,
author = {Mohammadi, Amir Reza and Peintner, Andreas and M\"{u}ller, Michael and Zangerle, Eva},
title = {Are We Explaining the Same Recommenders? Incorporating Recommender Performance for Evaluating Explainers},
year = {2024},
isbn = {9798400705052},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3640457.3691709},
doi = {10.1145/3640457.3691709},
abstract = {Explainability in recommender systems is both crucial and challenging. Among the state-of-the-art explanation strategies, counterfactual explanation provides intuitive and easily understandable insights into model predictions by illustrating how a small change in the input can lead to a different outcome. Recently, this approach has garnered significant attention, with various studies employing different metrics to evaluate the performance of these explanation methods. In this paper, we investigate the metrics used for evaluating counterfactual explainers for recommender systems. Through extensive experiments, we demonstrate that the performance of recommenders has a direct effect on counterfactual explainers and ignoring it results in inconsistencies in the evaluation results of explainer methods. Our findings highlight an additional challenge in evaluating counterfactual explainer methods and underscore the need to report the recommender performance or consider it in evaluation metrics.},
booktitle = {Proceedings of the 18th ACM Conference on Recommender Systems},
pages = {1113–1118},
numpages = {6},
keywords = {Counterfactual Explanation, Evaluation, Recommender Systems},
location = {Bari, Italy},
series = {RecSys '24}
}

  

@inproceedings{DBLP:conf/clef/ZangerleMP023,
  author       = {Eva Zangerle and
                  Maximilian Mayerl and
                  Martin Potthast and
                  Benno Stein},
  editor       = {Mohammad Aliannejadi and
                  Guglielmo Faggioli and
                  Nicola Ferro and
                  Michalis Vlachos},
  title        = {Overview of the Multi-Author Writing Style Analysis Task at {PAN}
                  2023},
  booktitle    = {Working Notes of the Conference and Labs of the Evaluation Forum {(CLEF}
                  2023), Thessaloniki, Greece, September 18th to 21st, 2023},
  series       = {{CEUR} Workshop Proceedings},
  volume       = {3497},
  pages        = {2513--2522},
  publisher    = {CEUR-WS.org},
  year         = {2023},
  url          = {https://ceur-ws.org/Vol-3497/paper-201.pdf},
  timestamp    = {Wed, 25 Oct 2023 17:24:28 +0200},
  biburl       = {https://dblp.org/rec/conf/clef/ZangerleMP023.bib},
  bibsource    = {dblp computer science bibliography, https://dblp.org}
}

@inproceedings{DBLP:conf/recsys/Peintner23,
  author       = {Andreas Peintner},
  title        = {Sequential Recommendation Models: {A} Graph-based Perspective},
  booktitle    = {Proceedings of the 17th {ACM} Conference on Recommender Systems, RecSys
                  2023, Singapore, Singapore, September 18-22, 2023},
  pages        = {1295--1299},
  publisher    = {{ACM}},
  year         = {2023},
  url          = {https://doi.org/10.1145/3604915.3608776},
  doi          = {10.1145/3604915.3608776},
  timestamp    = {Sun, 24 Sep 2023 15:46:34 +0200},
  biburl       = {https://dblp.org/rec/conf/recsys/Peintner23.bib},
  bibsource    = {dblp computer science bibliography, https://dblp.org}
}

@proceedings{DBLP:conf/recsys/2023perspectives,
  editor       = {Alan Said and
                  Eva Zangerle and
                  Christine Bauer},
  title        = {Proceedings of the 3rd Workshop Perspectives on the Evaluation of Recommender Systems 2023 co-located with the 17th {ACM} Conference on Recommender Systems (RecSys 2023), Singapore, Singapore, September
                  19, 2023},
  series       = {{CEUR} Workshop Proceedings},
  volume       = {3476},
  publisher    = {CEUR-WS.org},
  year         = {2023},
  url          = {https://ceur-ws.org/Vol-3476},
  urn          = {urn:nbn:de:0074-3476-4},
}

@inproceedings{DBLP:conf/recsys/Mohammadi23,
  author       = {Amir Reza Mohammadi},
  title        = {Explainable Graph Neural Network Recommenders; Challenges and Opportunities},
  booktitle    = {Proceedings of the 17th {ACM} Conference on Recommender Systems, RecSys
                  2023, Singapore, Singapore, September 18-22, 2023},
  pages        = {1318--1324},
  publisher    = {{ACM}},
  year         = {2023},
  url          = {https://doi.org/10.1145/3604915.3608875},
  doi          = {10.1145/3604915.3608875},
  timestamp    = {Sun, 24 Sep 2023 15:46:34 +0200},
  biburl       = {https://dblp.org/rec/conf/recsys/Mohammadi23.bib},
  bibsource    = {dblp computer science bibliography, https://dblp.org}
}

@inproceedings{DBLP:conf/recsys/PeintnerMZ23,
  author       = {Andreas Peintner and
                  Amir Reza Mohammadi and
                  Eva Zangerle},
  title        = {{SPARE:} Shortest Path Global Item Relations for Efficient Session-based
                  Recommendation},
  booktitle    = {Proceedings of the 17th {ACM} Conference on Recommender Systems, RecSys
                  2023, Singapore, Singapore, September 18-22, 2023},
  pages        = {58--69},
  publisher    = {{ACM}},
  year         = {2023},
  url          = {https://doi.org/10.1145/3604915.3608768},
  doi          = {10.1145/3604915.3608768}
}

@inproceedings{bevendorff:clef:2023,
author="Bevendorff, Janek
and Borrego-Obrador, Ian
and Chinea-R{\'i}os, Mara
and Franco-Salvador, Marc
and Fr{\"o}be, Maik
and Heini, Annina
and Kredens, Krzysztof
and Mayerl, Maximilian
and P{\k{e}}zik, Piotr
and Potthast, Martin
and Rangel, Francisco
and Rosso, Paolo
and Stamatatos, Efstathios
and Stein, Benno
and Wiegmann, Matti
and Wolska, Magdalena
and Zangerle, Eva",
title="Overview of PAN 2023: Authorship Verification, Multi-Author Writing Style Analysis, Profiling Cryptocurrency Influencers, and Trigger Detection",
booktitle="Experimental IR Meets Multilinguality, Multimodality, and Interaction",
year="2023",
publisher="Springer Nature Switzerland",
address="Cham",
pages="459--481",
abstract="The paper gives a brief overview of three shared tasks which have been organized at the PAN 2023 lab on digital text forensics and stylometry hosted at the CLEF 2023 conference. The tasks include authorship verification across discourse types, multi-author writing style analysis, profiling cryptocurrency influencers with few-shot learning, and trigger detection. Authorship verification and multi-author analysis continue and advance from past editions of PAN and influencer profiling and trigger detection are new tasks with novel research questions and evaluation resources. All four tasks alilgn with the goals of all shared tasks at PAN: to advance the state of the art in text forensics and stylometry while ensuring objective evaluation on newly developed benchmark datasets.",
isbn="978-3-031-42448-9",
doi="10.1007/978-3-031-42448-9_29"
}

@inproceedings{DBLP:conf/recsys/SaidZ023,
  author       = {Alan Said and
                  Eva Zangerle and
                  Christine Bauer},
  title        = {Third Workshop: Perspectives on the Evaluation of Recommender Systems
                  {(PERSPECTIVES} 2023)},
  booktitle    = {Proceedings of the 17th {ACM} Conference on Recommender Systems, RecSys
                  2023, Singapore, Singapore, September 18-22, 2023},
  pages        = {1221--1222},
  publisher    = {{ACM}},
  year         = {2023},
  url          = {https://doi.org/10.1145/3604915.3608748},
  doi          = {10.1145/3604915.3608748},
}

@article{10.1145/3556536,
author = {Zangerle, Eva and Bauer, Christine},
title = {Evaluating Recommender Systems: Survey and Framework},
year = {2023},
month = {August},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
issn = {0360-0300},
url = {https://doi.org/10.1145/3556536},
doi = {10.1145/3556536},
abstract = {The comprehensive evaluation of the performance of a recommender system is a complex endeavor: many facets need to be considered in configuring an adequate and effective evaluation setting. Such facets include, for instance, defining the specific goals of the evaluation, choosing an evaluation method, underlying data, and suitable evaluation metrics. In this paper, we consolidate and systematically organize this dispersed knowledge on recommender systems evaluation. We introduce the “Framework for EValuating Recommender systems” (FEVR) that we derive from the discourse on recommender systems evaluation. In FEVR, we categorize the evaluation space of recommender systems evaluation. We postulate that the comprehensive evaluation of a recommender system frequently requires considering multiple facets and perspectives in the evaluation. The FEVR framework provides a structured foundation to adopt adequate evaluation configurations that encompass this required multi-facettedness and provides the basis to advance in the field. We outline and discuss the challenges of a comprehensive evaluation of recommender systems, and provide an outlook on what we need to embrace and do to move forward as a research community.},
journal = {ACM Computing Surveys},
volume       = 55,
number       = 8,
articleno = {170},
numpages = {38},
keywords = {Framework for EValuating Recommender systems, survey, FEVR}
}