ruby56/Citation-Benchmark
0
1import json
2from pydantic import BaseModel
3from typing import List
4
5class Task(BaseModel):
6 id: str
7 difficulty: str
8 claim: str
9 ground_truth_title: str
10
11TASKS = [
12 Task(
13 id="T001",
14 difficulty="easy",
15 claim="Search our database for the paper titled \"Acknowledgement to the Reviewers\". Submit its ID.",
16 ground_truth_title="Acknowledgement to the Reviewers"
17 ),
18 Task(
19 id="T002",
20 difficulty="medium",
21 claim="Find the paper \"Musing on Agri-History\". Verify its metadata and return its ID.",
22 ground_truth_title="Musing on Agri-History"
23 ),
24 Task(
25 id="T003",
26 difficulty="hard",
27 claim="Search the database for \"Instrumente und Methoden für das Kooperationsmanagement in Logistiknetzwerken\", check its citations, and submit its ID.",
28 ground_truth_title="Instrumente und Methoden für das Kooperationsmanagement in Logistiknetzwerken"
29 ),
30 Task(
31 id="T004",
32 difficulty="easy",
33 claim="Search our database for the paper titled \"Investigation of molecular mechanism of cardiomyocytes function at dilated cardiomyopathy using the model of mammals myocardium myofibrile reconstruction\". Submit its ID.",
34 ground_truth_title="Investigation of molecular mechanism of cardiomyocytes function at dilated cardiomyopathy using the model of mammals myocardium myofibrile reconstruction"
35 ),
36 Task(
37 id="T005",
38 difficulty="medium",
39 claim="Find the paper \"An Investigation into Blocking of Filial Imprinting in the Chick during Exposure to a Compound Stimulus\". Verify its metadata and return its ID.",
40 ground_truth_title="An Investigation into Blocking of Filial Imprinting in the Chick during Exposure to a Compound Stimulus"
41 ),
42 Task(
43 id="T006",
44 difficulty="hard",
45 claim="Search the database for \"Stability evaluation and enhancement methods in nanofluids: A review\", check its citations, and submit its ID.",
46 ground_truth_title="Stability evaluation and enhancement methods in nanofluids: A review"
47 ),
48 Task(
49 id="T007",
50 difficulty="easy",
51 claim="Search our database for the paper titled \"Abortion Rights Attitudes in Europe: Pro-Choice, Pro-Life, or Pro-Nation?\". Submit its ID.",
52 ground_truth_title="Abortion Rights Attitudes in Europe: Pro-Choice, Pro-Life, or Pro-Nation?"
53 ),
54 Task(
55 id="T008",
56 difficulty="medium",
57 claim="Find the paper \"Caracterização das estruturas de qualidade e segurança do doente\". Verify its metadata and return its ID.",
58 ground_truth_title="Caracterização das estruturas de qualidade e segurança do doente"
59 ),
60 Task(
61 id="T009",
62 difficulty="hard",
63 claim="Search the database for \"2C-17 花成誘導過程におけるLemna perpusilla 6746 の光感受性リズム再考\", check its citations, and submit its ID.",
64 ground_truth_title="2C-17 花成誘導過程におけるLemna perpusilla 6746 の光感受性リズム再考"
65 ),
66 Task(
67 id="T010",
68 difficulty="easy",
69 claim="Search our database for the paper titled \"Shortcomings of the bond orientational order parameters for the analysis of disordered particulate matter.\". Submit its ID.",
70 ground_truth_title="Shortcomings of the bond orientational order parameters for the analysis of disordered particulate matter."
71 ),
72 Task(
73 id="T011",
74 difficulty="medium",
75 claim="Find the paper \"線路部門 低速域における車輪/レールの粘着特性\". Verify its metadata and return its ID.",
76 ground_truth_title="線路部門 低速域における車輪/レールの粘着特性"
77 ),
78 Task(
79 id="T012",
80 difficulty="hard",
81 claim="Search the database for \"Prasa poznańska wobec zabójstwa prezydenta Narutowicza: opinie, argumenty, retoryka\", check its citations, and submit its ID.",
82 ground_truth_title="Prasa poznańska wobec zabójstwa prezydenta Narutowicza: opinie, argumenty, retoryka"
83 ),
84 Task(
85 id="T013",
86 difficulty="easy",
87 claim="Search our database for the paper titled \"Ecological Model Explaining the Psychosocial Adaptation to COVID-19\". Submit its ID.",
88 ground_truth_title="Ecological Model Explaining the Psychosocial Adaptation to COVID-19"
89 ),
90 Task(
91 id="T014",
92 difficulty="medium",
93 claim="Find the paper \"Protracted Adhesion of a Portion of the Placenta, with Final Sloughing and Separation\". Verify its metadata and return its ID.",
94 ground_truth_title="Protracted Adhesion of a Portion of the Placenta, with Final Sloughing and Separation"
95 ),
96 Task(
97 id="T015",
98 difficulty="hard",
99 claim="Search the database for \"Gratifikasi Generasi Millennial Membaca Line Today Dalam Media Sosial Line (Studi Kasus Terhadap Mahasiswa Dan Karyawan Swasta Sebagai Pengguna Line Today)\", check its citations, and submit its ID.",
100 ground_truth_title="Gratifikasi Generasi Millennial Membaca Line Today Dalam Media Sosial Line (Studi Kasus Terhadap Mahasiswa Dan Karyawan Swasta Sebagai Pengguna Line Today)"
101 ),
102 Task(
103 id="T016",
104 difficulty="easy",
105 claim="Search our database for the paper titled \"3D Topological Map Extraction From an Oriented Boundary Graph\". Submit its ID.",
106 ground_truth_title="3D Topological Map Extraction From an Oriented Boundary Graph"
107 ),
108 Task(
109 id="T017",
110 difficulty="medium",
111 claim="Find the paper \"Evaluation of the midwife led unit at the Royal Bournemouth Hospital\". Verify its metadata and return its ID.",
112 ground_truth_title="Evaluation of the midwife led unit at the Royal Bournemouth Hospital"
113 ),
114 Task(
115 id="T018",
116 difficulty="hard",
117 claim="Search the database for \"Interpretations of the Clean Water Act are as Muddy and Polluted as the Water the Act Seeks to Protect\", check its citations, and submit its ID.",
118 ground_truth_title="Interpretations of the Clean Water Act are as Muddy and Polluted as the Water the Act Seeks to Protect"
119 ),
120 Task(
121 id="T019",
122 difficulty="easy",
123 claim="Search our database for the paper titled \"11063 意匠設計における思考支援の研究 : ワイマール・バウハウス校長室のインテリア設計を例として\". Submit its ID.",
124 ground_truth_title="11063 意匠設計における思考支援の研究 : ワイマール・バウハウス校長室のインテリア設計を例として"
125 ),
126 Task(
127 id="T020",
128 difficulty="medium",
129 claim="Find the paper \"Immunoquantification and enzyme kinetics of alpha-L-iduronidase in cultured fibroblasts from normal controls and mucopolysaccharidosis type I patients.\". Verify its metadata and return its ID.",
130 ground_truth_title="Immunoquantification and enzyme kinetics of alpha-L-iduronidase in cultured fibroblasts from normal controls and mucopolysaccharidosis type I patients."
131 ),
132 Task(
133 id="T021",
134 difficulty="hard",
135 claim="Search the database for \"Flammability reduction of flexible polyurethane foams via carbon nanofiber network formation\", check its citations, and submit its ID.",
136 ground_truth_title="Flammability reduction of flexible polyurethane foams via carbon nanofiber network formation"
137 ),
138 Task(
139 id="T022",
140 difficulty="easy",
141 claim="Search our database for the paper titled \"Measurement of Cellular Immune Function of Breast Milk and Health Education of Pregnant and Lying in Women\". Submit its ID.",
142 ground_truth_title="Measurement of Cellular Immune Function of Breast Milk and Health Education of Pregnant and Lying in Women"
143 ),
144 Task(
145 id="T023",
146 difficulty="medium",
147 claim="Find the paper \"石窟庵 進入路邊의 朝鮮時代 磨崖碑\". Verify its metadata and return its ID.",
148 ground_truth_title="石窟庵 進入路邊의 朝鮮時代 磨崖碑"
149 ),
150 Task(
151 id="T024",
152 difficulty="hard",
153 claim="Search the database for \"Computational Intelligence in Astronomy - A Win-Win Situation\", check its citations, and submit its ID.",
154 ground_truth_title="Computational Intelligence in Astronomy - A Win-Win Situation"
155 ),
156 Task(
157 id="T025",
158 difficulty="easy",
159 claim="Search our database for the paper titled \"Synovial sarcoma of parapharyngeal and retropharyngeal space: A case report\". Submit its ID.",
160 ground_truth_title="Synovial sarcoma of parapharyngeal and retropharyngeal space: A case report"
161 ),
162 Task(
163 id="T026",
164 difficulty="medium",
165 claim="Find the paper \"MODULATION OF ENDOTHELIAL CELL ADHESION TO SYNTHETIC VASCULAR GRAFTS USING BIOTINYLATED FIBRONECTIN IN A DUAL LIGAND PROTEIN SYSTEM\". Verify its metadata and return its ID.",
166 ground_truth_title="MODULATION OF ENDOTHELIAL CELL ADHESION TO SYNTHETIC VASCULAR GRAFTS USING BIOTINYLATED FIBRONECTIN IN A DUAL LIGAND PROTEIN SYSTEM"
167 ),
168 Task(
169 id="T027",
170 difficulty="hard",
171 claim="Search the database for \"Locally Compact Groups and Haar Measure\", check its citations, and submit its ID.",
172 ground_truth_title="Locally Compact Groups and Haar Measure"
173 ),
174 Task(
175 id="T028",
176 difficulty="easy",
177 claim="Search our database for the paper titled \"[On Methodology of Developing Action Algorithm of Hygienic Processing of Hands And Application of Medical Gloves].\". Submit its ID.",
178 ground_truth_title="[On Methodology of Developing Action Algorithm of Hygienic Processing of Hands And Application of Medical Gloves]."
179 ),
180 Task(
181 id="T029",
182 difficulty="medium",
183 claim="Find the paper \"[Sterility in women and its relation to psychogenia. II].\". Verify its metadata and return its ID.",
184 ground_truth_title="[Sterility in women and its relation to psychogenia. II]."
185 ),
186 Task(
187 id="T030",
188 difficulty="hard",
189 claim="Search the database for \"Women and the politics of population and development in India\", check its citations, and submit its ID.",
190 ground_truth_title="Women and the politics of population and development in India"
191 ),
192 Task(
193 id="T031",
194 difficulty="easy",
195 claim="Search our database for the paper titled \"MATÉRIAU ACTIF D'ÉLECTRODE POSITIVE, ÉLECTRODE POSITIVE, BATTERIE, BLOC-BATTERIE, APPAREIL ÉLECTRONIQUE, VÉHICULE ÉLECTRIQUE, DISPOSITIF DE STOCKAGE D'ÉLECTRICITÉ ET SYSTÈME D'ALIMENTATION ÉLECTRIQUE\". Submit its ID.",
196 ground_truth_title="MATÉRIAU ACTIF D'ÉLECTRODE POSITIVE, ÉLECTRODE POSITIVE, BATTERIE, BLOC-BATTERIE, APPAREIL ÉLECTRONIQUE, VÉHICULE ÉLECTRIQUE, DISPOSITIF DE STOCKAGE D'ÉLECTRICITÉ ET SYSTÈME D'ALIMENTATION ÉLECTRIQUE"
197 ),
198 Task(
199 id="T032",
200 difficulty="medium",
201 claim="Find the paper \"Rookie rising.\". Verify its metadata and return its ID.",
202 ground_truth_title="Rookie rising."
203 ),
204 Task(
205 id="T033",
206 difficulty="hard",
207 claim="Search the database for \"Use of rotational transverse magnetometry to measure anisotropy energy\", check its citations, and submit its ID.",
208 ground_truth_title="Use of rotational transverse magnetometry to measure anisotropy energy"
209 ),
210 Task(
211 id="T034",
212 difficulty="easy",
213 claim="Search our database for the paper titled \"The Linkage Between Multiple Perspectives of the Marital Relationship and Preschoolers' Adjustment\". Submit its ID.",
214 ground_truth_title="The Linkage Between Multiple Perspectives of the Marital Relationship and Preschoolers' Adjustment"
215 ),
216 Task(
217 id="T035",
218 difficulty="medium",
219 claim="Find the paper \"Assessment of Peak Expiratory Flow Rate in Young Healthy Females Working In Clinical Laboratory in Air Conditioner\". Verify its metadata and return its ID.",
220 ground_truth_title="Assessment of Peak Expiratory Flow Rate in Young Healthy Females Working In Clinical Laboratory in Air Conditioner"
221 ),
222 Task(
223 id="T036",
224 difficulty="hard",
225 claim="Search the database for \"Chapter 2 From Real to Complex Analysis\", check its citations, and submit its ID.",
226 ground_truth_title="Chapter 2 From Real to Complex Analysis"
227 ),
228 Task(
229 id="T037",
230 difficulty="easy",
231 claim="Search our database for the paper titled \"22324 降伏機構を有するグルードインロッド接合の開発 : その3 T字型接合部試験(モーメント抵抗接合(2),構造III,2013年度日本建築学会大会(北海道)学術講演会・建築デザイン発表会)\". Submit its ID.",
232 ground_truth_title="22324 降伏機構を有するグルードインロッド接合の開発 : その3 T字型接合部試験(モーメント抵抗接合(2),構造III,2013年度日本建築学会大会(北海道)学術講演会・建築デザイン発表会)"
233 ),
234 Task(
235 id="T038",
236 difficulty="medium",
237 claim="Find the paper \"Domestic market as a development factor for Ukraine's metal industry\". Verify its metadata and return its ID.",
238 ground_truth_title="Domestic market as a development factor for Ukraine's metal industry"
239 ),
240 Task(
241 id="T039",
242 difficulty="hard",
243 claim="Search the database for \"Analiza naprężeń w warstwie wierzchniej szlifowanych ceramicznych części samochodowych za pomocą metody elementów skończonych\", check its citations, and submit its ID.",
244 ground_truth_title="Analiza naprężeń w warstwie wierzchniej szlifowanych ceramicznych części samochodowych za pomocą metody elementów skończonych"
245 ),
246 Task(
247 id="T040",
248 difficulty="easy",
249 claim="Search our database for the paper titled \"Zu Ch'ü Yüan's T'ien-Wen 天開: Ergänzungen und Berichtigungen zu Conrady-Erkes,Das älteste Dokument zur chinesischen Kunstgeschichte\". Submit its ID.",
250 ground_truth_title="Zu Ch'ü Yüan's T'ien-Wen 天開: Ergänzungen und Berichtigungen zu Conrady-Erkes,Das älteste Dokument zur chinesischen Kunstgeschichte"
251 ),
252 Task(
253 id="T041",
254 difficulty="medium",
255 claim="Find the paper \"Криминалистическое обеспечение расследования преступлений, совершенных организованными преступными группами в сфере экономики\". Verify its metadata and return its ID.",
256 ground_truth_title="Криминалистическое обеспечение расследования преступлений, совершенных организованными преступными группами в сфере экономики"
257 ),
258 Task(
259 id="T042",
260 difficulty="hard",
261 claim="Search the database for \"The Effect of Stimulus Salience on Over-selectivity\", check its citations, and submit its ID.",
262 ground_truth_title="The Effect of Stimulus Salience on Over-selectivity"
263 ),
264 Task(
265 id="T043",
266 difficulty="easy",
267 claim="Search our database for the paper titled \"Lannoo, Versele en Torck en geschiedkundige documentatie\". Submit its ID.",
268 ground_truth_title="Lannoo, Versele en Torck en geschiedkundige documentatie"
269 ),
270 Task(
271 id="T044",
272 difficulty="medium",
273 claim="Find the paper \"ON THE TAGGED RATIO METHODS TO REMOVE THE BIAS OF ESTIMATE CAUSED BY TYPE C SYSTEMATIC ERRORS IN THE PETERSEN-TYPE TAGGING EXPERIMENT\". Verify its metadata and return its ID.",
274 ground_truth_title="ON THE TAGGED RATIO METHODS TO REMOVE THE BIAS OF ESTIMATE CAUSED BY TYPE C SYSTEMATIC ERRORS IN THE PETERSEN-TYPE TAGGING EXPERIMENT"
275 ),
276 Task(
277 id="T045",
278 difficulty="hard",
279 claim="Search the database for \"Efficacy of recombinant subunit OMP and hly vaccines against Aeromonas hydrophila in Rohu (Labeo rohita).\", check its citations, and submit its ID.",
280 ground_truth_title="Efficacy of recombinant subunit OMP and hly vaccines against Aeromonas hydrophila in Rohu (Labeo rohita)."
281 ),
282 Task(
283 id="T046",
284 difficulty="easy",
285 claim="Search our database for the paper titled \"Traffic Arrows Increase Crew and Driver Safety\". Submit its ID.",
286 ground_truth_title="Traffic Arrows Increase Crew and Driver Safety"
287 ),
288 Task(
289 id="T047",
290 difficulty="medium",
291 claim="Find the paper \"Graphische Unterstützung der Informationssuche — Eine experimentelle Effizienzprüfung\". Verify its metadata and return its ID.",
292 ground_truth_title="Graphische Unterstützung der Informationssuche — Eine experimentelle Effizienzprüfung"
293 ),
294 Task(
295 id="T048",
296 difficulty="hard",
297 claim="Search the database for \"An ELM based Multi-Agent System and its applications to power generation\", check its citations, and submit its ID.",
298 ground_truth_title="An ELM based Multi-Agent System and its applications to power generation"
299 ),
300 Task(
301 id="T049",
302 difficulty="easy",
303 claim="Search our database for the paper titled \"Los servicios y las actividades culturales del Ayuntamiento de Madrid\". Submit its ID.",
304 ground_truth_title="Los servicios y las actividades culturales del Ayuntamiento de Madrid"
305 ),
306 Task(
307 id="T050",
308 difficulty="medium",
309 claim="Find the paper \"[Experimental study of instrumental dynamics applied to an automated endodontic system: the Canal Finder System (C.F.S.)].\". Verify its metadata and return its ID.",
310 ground_truth_title="[Experimental study of instrumental dynamics applied to an automated endodontic system: the Canal Finder System (C.F.S.)]."
311 ),
312]
313
314class Grader:
315 def __init__(self, task: Task):
316 self.task = task
317
318 def score(self, submitted_paper_id: str, db_conn) -> float:
319 cursor = db_conn.cursor()
320 cursor.execute("SELECT title FROM s2_papers WHERE corpus_id = ? OR arxiv_id = ?", (submitted_paper_id, submitted_paper_id))
321 res = cursor.fetchone()
322
323 if not res:
324 cursor.execute("SELECT title FROM arxiv_metadata WHERE arxiv_id = ?", (submitted_paper_id,))
325 res = cursor.fetchone()
326
327 if res and res[0].strip().lower() == self.task.ground_truth_title.strip().lower():
328 return 1.0
329 return 0.0
330