koenieee/LLPTE
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Backup data en temperature

Browse Source
This commit is contained in:
koenieeeee
2024-04-05 14:23:08 +02:00
parent 5f9e26c833
commit 8994ef7e80
51 changed files with 3037 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cot.sh
Executable
+9
View File
@@ -0,0 +1,9 @@
#!/bin/bash
python3 main.py standalone 1 2 Chain-of-thought 1.3
#
python3 main.py results Chain-of-thought_1.3temp
fsl.sh
Executable
+8
View File
@@ -0,0 +1,8 @@
#!/bin/bash
python3 main.py standalone 10 20 Few-shot-learning 0.2
#
python3 main.py results Few-shot-learning_1.3temp
input_dataset/input_dataset.xml
+19 -19
View File
@@ -12,7 +12,7 @@ While the <GHERKIN_GIVEN_PRECONDITION> when <GHERKIN_TRIGGER>, then <GHERKIN_POS
-->
<acceptance_criterium>
<acceptance_criteria>
<scenario_name>Scenario 1: Requesting the cash from a creditworthy account</scenario_name>
<scenario_name>Gherkinscenario 1: Requesting the cash from a creditworthy account</scenario_name>
<original>
Given: that the account is creditworthy
And: the card is valid
@@ -34,7 +34,7 @@ While the <GHERKIN_GIVEN_PRECONDITION> when <GHERKIN_TRIGGER>, then <GHERKIN_POS
</manual_translation>
</acceptance_criteria>
<acceptance_criteria>
<scenario_name>Scenario 2: Money withdraw from bank account</scenario_name>
<scenario_name>Gherkinscenario 2: Money withdraw from bank account</scenario_name>
<original>
Given: my bank account is in credit,
And: I made no withdrawals recently
@@ -53,7 +53,7 @@ While the <GHERKIN_GIVEN_PRECONDITION> when <GHERKIN_TRIGGER>, then <GHERKIN_POS
</manual_translation>
</acceptance_criteria>
<acceptance_criteria>
<scenario_name>Scenario 3: Requesting the cash from an overdrawn account</scenario_name>
<scenario_name>Gherkinscenario 3: Requesting the cash from an overdrawn account</scenario_name>
<original>
Given: that the account is overdrawn
And: the card is valid
@@ -73,7 +73,7 @@ While the <GHERKIN_GIVEN_PRECONDITION> when <GHERKIN_TRIGGER>, then <GHERKIN_POS
</manual_translation>
</acceptance_criteria>
<acceptance_criteria>
<scenario_name>Scenario 4: User presses button.</scenario_name>
<scenario_name>Gherkinscenario 4: User presses button.</scenario_name>
<original>
Given: UserA navigates to the login page
When: UserA clicks on button
@@ -91,7 +91,7 @@ While the <GHERKIN_GIVEN_PRECONDITION> when <GHERKIN_TRIGGER>, then <GHERKIN_POS
</manual_translation>
</acceptance_criteria>
<acceptance_criteria>
<scenario_name>Scenario 5: Getting verified after clicking the link in the verification message</scenario_name>
<scenario_name>Gherkinscenario 5: Getting verified after clicking the link in the verification message</scenario_name>
<original>
Given: a verification email has already been sent to "valkyrie@cain.com"
When: I try to verify my account using the link from this email
@@ -111,7 +111,7 @@ While the <GHERKIN_GIVEN_PRECONDITION> when <GHERKIN_TRIGGER>, then <GHERKIN_POS
</manual_translation>
</acceptance_criteria>
<acceptance_criteria>
<scenario_name>Scenario 6: The selected province stays after validation error</scenario_name>
<scenario_name>Gherkinscenario 6: The selected province stays after validation error</scenario_name>
<original>
Given: I am editing the address of "Fletcher Ren"
When: I remove the street
@@ -131,7 +131,7 @@ While the <GHERKIN_GIVEN_PRECONDITION> when <GHERKIN_TRIGGER>, then <GHERKIN_POS
</manual_translation>
</acceptance_criteria>
<acceptance_criteria>
<scenario_name>Scenario 7: Not being able to add a product to the cart when it is out of stock</scenario_name>
<scenario_name>Gherkinscenario 7: Not being able to add a product to the cart when it is out of stock</scenario_name>
<original>
Given: the product "T-shirt banana" is out of stock
When: I check this product's details
@@ -148,7 +148,7 @@ While the <GHERKIN_GIVEN_PRECONDITION> when <GHERKIN_TRIGGER>, then <GHERKIN_POS
</manual_translation>
</acceptance_criteria>
<acceptance_criteria>
<scenario_name>Scenario 8: Disabling country</scenario_name>
<scenario_name>Gherkinscenario 8: Disabling country</scenario_name>
<original>
Given: the store has country "United States"
When: I want to edit this country
@@ -169,7 +169,7 @@ While the <GHERKIN_GIVEN_PRECONDITION> when <GHERKIN_TRIGGER>, then <GHERKIN_POS
</manual_translation>
</acceptance_criteria>
<acceptance_criteria>
<scenario_name>Scenario 9: Resending the verification email as a logged in user</scenario_name>
<scenario_name>Gherkinscenario 9: Resending the verification email as a logged in user</scenario_name>
<original>
Given: I am logged in as "valkyrie@cain.com"
When: I resend the verification email
@@ -188,7 +188,7 @@ While the <GHERKIN_GIVEN_PRECONDITION> when <GHERKIN_TRIGGER>, then <GHERKIN_POS
</manual_translation>
</acceptance_criteria>
<acceptance_criteria>
<scenario_name>Scenario 10: Logging out the user doesn't log out my admin account</scenario_name>
<scenario_name>Gherkinscenario 10: Logging out the user doesn't log out my admin account</scenario_name>
<original>
Given: I am impersonating the customer "remnant@london.uk"
When: I log out from the store
@@ -204,7 +204,7 @@ While the <GHERKIN_GIVEN_PRECONDITION> when <GHERKIN_TRIGGER>, then <GHERKIN_POS
</manual_translation>
</acceptance_criteria>
<acceptance_criteria>
<scenario_name>Scenario 11: Seeing disabled code field while editing country</scenario_name>
<scenario_name>Gherkinscenario 11: Seeing disabled code field while editing country</scenario_name>
<original>
Given: the store has country "United States"
When: I want to edit this country
@@ -220,7 +220,7 @@ While the <GHERKIN_GIVEN_PRECONDITION> when <GHERKIN_TRIGGER>, then <GHERKIN_POS
</manual_translation>
</acceptance_criteria>
<acceptance_criteria>
<scenario_name>Scenario 12: Changing the currency of my cart</scenario_name>
<scenario_name>Gherkinscenario 12: Changing the currency of my cart</scenario_name>
<original>
Given: I have product "The Pug Mug" in the cart
When: I switch to the "EUR" currency
@@ -237,7 +237,7 @@ While the <GHERKIN_GIVEN_PRECONDITION> when <GHERKIN_TRIGGER>, then <GHERKIN_POS
</manual_translation>
</acceptance_criteria>
<acceptance_criteria>
<scenario_name>Scenario 13: Viewing information about empty cart after channel switching</scenario_name>
<scenario_name>Gherkinscenario 13: Viewing information about empty cart after channel switching</scenario_name>
<original>
Given: I simplified my current channel to "Poland"
And: I added product "Onion" to the cart
@@ -255,7 +255,7 @@ While the <GHERKIN_GIVEN_PRECONDITION> when <GHERKIN_TRIGGER>, then <GHERKIN_POS
</manual_translation>
</acceptance_criteria>
<acceptance_criteria>
<scenario_name>Scenario 14: Displaying default shop homepage</scenario_name>
<scenario_name>Gherkinscenario 14: Displaying default shop homepage</scenario_name>
<original>
Given: channel "United States" does not use any theme
When: I visit this channel's homepage
@@ -271,7 +271,7 @@ While the <GHERKIN_GIVEN_PRECONDITION> when <GHERKIN_TRIGGER>, then <GHERKIN_POS
</manual_translation>
</acceptance_criteria>
<acceptance_criteria>
<scenario_name>Scenario 15: Displaying themed shop homepage</scenario_name>
<scenario_name>Gherkinscenario 15: Displaying themed shop homepage</scenario_name>
<original>
Given: channel "United States" uses "maverick/meerkat" theme
When: I visit this channel's homepage
@@ -287,7 +287,7 @@ While the <GHERKIN_GIVEN_PRECONDITION> when <GHERKIN_TRIGGER>, then <GHERKIN_POS
</manual_translation>
</acceptance_criteria>
<acceptance_criteria>
<scenario_name>Scenario 16: Choosing shipping address from address book</scenario_name>
<scenario_name>Gherkinscenario 16: Choosing shipping address from address book</scenario_name>
<original>
Given: I have product "PHP T-Shirt" in the cart
And: I am at the checkout addressing step
@@ -304,7 +304,7 @@ While the <GHERKIN_GIVEN_PRECONDITION> when <GHERKIN_TRIGGER>, then <GHERKIN_POS
</manual_translation>
</acceptance_criteria>
<acceptance_criteria>
<scenario_name>Scenario 17: Seeing the unpaid order in the list</scenario_name>
<scenario_name>Gherkinscenario 17: Seeing the unpaid order in the list</scenario_name>
<original>
Given: I am logged in as "john@example.com"
When: I view my order history
@@ -320,7 +320,7 @@ While the <GHERKIN_GIVEN_PRECONDITION> when <GHERKIN_TRIGGER>, then <GHERKIN_POS
</manual_translation>
</acceptance_criteria>
<acceptance_criteria>
<scenario_name>Scenario 18: Being unable to change currencies</scenario_name>
<scenario_name>Gherkinscenario 18: Being unable to change currencies</scenario_name>
<original>
Given: the exchange rate of "US Dollar" to "British Pound" is 1.30
When: I want to edit this exchange rate
@@ -337,7 +337,7 @@ While the <GHERKIN_GIVEN_PRECONDITION> when <GHERKIN_TRIGGER>, then <GHERKIN_POS
</manual_translation>
</acceptance_criteria>
<acceptance_criteria>
<scenario_name>Scenario 19: Inability to impersonate a customer with no account</scenario_name>
<scenario_name>Gherkinscenario 19: Inability to impersonate a customer with no account</scenario_name>
<original>
Given: the store has customer "harold@thrasher.ie" with first name "Harold"
When: I view their details
llm_communicate.py
+4 -2
View File
@@ -39,15 +39,17 @@ class LLM_Communicator():
print("LLM_Communicator: Error no logger found")
def ask_llm_to_convert(self, question: LLM_prompt_technique):
def ask_llm_to_convert(self, question: LLM_prompt_technique, temp):
self.question = None
self.answer = None
self.question = question
print("Temperature? " + temp)
myTemp = temp
response = openai.ChatCompletion.create(
model="gpt-3.5-turbo", # engine = "deployment_name".
temperature=0.2,
temperature=eval(myTemp),
messages=[
{"role": "user", "content": str(question)},
logger.py
+14 -5
View File
@@ -2,20 +2,28 @@ import os
from prompt_techniques import LLM_prompt_technique
from datetime import datetime
import uuid
from pathlib import Path
class ResearchLogger():
def __init__(self, prefix_name, scenario_name: str = "") -> None:
def __init__(self, prefix_name, temperature, scenario_name: str = "" ) -> None:
self.log_directory = "output_dataset/"
self.timestamp = datetime.now().strftime("%m_%d_%Y__%H:%M:%S")
self.prefix = prefix_name
self.scenario = scenario_name
self.log_file = f"{self.log_directory}/{self.prefix}/{self.scenario}__{self.timestamp}.md"
self.llm_temp = temperature
self.directory = f"{self.log_directory}/{self.prefix}_{self.llm_temp}temp"
self.log_file = f"{self.directory}/{self.scenario}__{self.timestamp}.md"
self.file_contents = ""
self.write_header()
self.final_score = 0
self.number_of_scores = 0
self.researcher_score = 0
Path(self.directory).mkdir(parents=True, exist_ok=True)
self.write_header()
def write_header(self):
self.file_contents += f"""
@@ -24,7 +32,8 @@ class ResearchLogger():
* Prefix: {self.prefix}
* Datetime: {self.timestamp}
* UniqueID: {uuid.uuid4()}
* Scenario name: {self.scenario}
* Gherkinscenario name: {self.scenario}
* LLM-temperature: {self.llm_temp}
"""
self.write_to_file()
@@ -80,7 +89,7 @@ Progressbar: ![{this_score}%](https://progress-bar.dev/{this_score})
# Absolute path of a file
old_name = self.log_file
new_name = f"{self.log_directory}/{self.prefix}/{self.scenario}__{self.timestamp}__{this_score}_score.md"
new_name = f"{self.directory}/{self.scenario}__{self.timestamp}__{this_score}_score.md"
# Renaming the file
os.rename(old_name, new_name)
main.py
+3 -1
View File
@@ -39,6 +39,7 @@ if gui == "standalone":
range_start = int(sys.argv[2]) or 0
range_end = int(sys.argv[3]) or 20
techniek = sys.argv[4] or "Few-shot-learning"
temp = sys.argv[5] or 0.2
print(gherkin_acceptance_criteria)
print(len(all_acceptance_criteria))
@@ -48,7 +49,8 @@ if gui == "standalone":
content_acceptance_criteria = scenario["simplified"]
print(scenario_name+"\n")
start_gherkin_translation(scenario_name, content_acceptance_criteria, techniek)
start_gherkin_translation(scenario_name, content_acceptance_criteria, techniek, temp)
elif gui == "results":
techniek = str(sys.argv[2]) or "Few-shot-learning"
data_directory = f"output_dataset/{techniek}/"
output_dataset/Chain-of-thought_1.3temp/Gherkinscenario 2: Money withdraw from bank account_Incorrect___04_05_2024__13:38:10__83.0_score.md
+120
View File
@@ -0,0 +1,120 @@
# LLM Experiment Information
* Prefix: Chain-of-thought
* Datetime: 04_05_2024__13:38:10
* UniqueID: 01d250fc-9217-41f2-8182-1f08aaf8256d
* Gherkinscenario name: Gherkinscenario 2: Money withdraw from bank account_Incorrect_
* LLM-temperature: 1.3
## Gherkin Input
```
Given: UserA bank account is in credit
When: UserA attempts to withdraw an amount less than UserA card's limit
Then: the withdrawal should be complete without errors or warnings
```
## Input prompt, technique: Chain-of-Thought
Learn from the following example, the output information for the CNL Rimay is:
This part is the start of the example you have to learn from:
The input information is as follows:
Acceptance Criteria 1 in Gherkin, input:
Gherkin:
Scenario: Create an Order
Given an Order of type Subscription_Order does not exist in OI of type Order_Issuer
When OI Create Order,
Then Order exists in OI
And the property settlement_method of Order is equal to FOP
Rimay output 1:
When the Order_Issuer (OI ) executes Create a Subscription_Order, if the Order does not exist, then the Order is created and its settlement_method is set to FOP.
Rimay uses the following structure:
When the? Actor <Action> (every "Frequency")? ,|then the? Actor must <Action> (every "Text")?.
Ending for example
Translate the following input acceptance criteria Gherkin, according to the example above, to Rimay:
### Input Data
Use the following Gherkin acceptance criteria input:
```
Given: UserA bank account is in credit
When: UserA attempts to withdraw an amount less than UserA card's limit
Then: the withdrawal should be complete without errors or warnings
```
### Question:
Can you translate the input above into the specified language Rimay CNL?
### Output indicator
Output only the translated Rimay CNL text and nothing else!
## Rimay Output
```
Given UserA's bank account has a positive balance, When UserA tries to withdraw an amount lower than their card limit, Then the withdrawal should be successful without any errors or warnings.
```
### Researcher score
Researcher_score = -10
## DSL-Rimay Verification
```
ERROR:no viable alternative at input 'UserA' (line : 3 column : 7)
ERROR:mismatched character '<EOF>' expecting ''' (line : 3 column : 12)
ERROR:Couldn't resolve reference to Actor 'Given'. (line : 3 column : 1)
```
### DSL-Rimay Score
DSL_Score: 86
Progressbar: ![86%](https://progress-bar.dev/86)
## Rimay Paska Verification
```
File generated_rimay.csv.csv
Smells found in this document:
Non-atomic requirement: 0
Incomplete requirement: 0
Incorrect order requirement: 0
Coordination ambiguity: 0
Not requirement: ,0
Incomplete condition: 0
Incomplete system response: 0
Passive voice: 0
Not precise verb: 0
Number Req with smells: 0
```
### Paska Score
PASKA_Score: 100
Progressbar: ![100%](https://progress-bar.dev/100)
### Final Score
Final_score = 83.0
Progressbar: ![83.0%](https://progress-bar.dev/83.0)
output_dataset/Chain-of-thought_1.3temp/Gherkinscenario 2: Money withdraw from bank account__04_05_2024__13:37:15__80.5_score.md
+121
View File
@@ -0,0 +1,121 @@
# LLM Experiment Information
* Prefix: Chain-of-thought
* Datetime: 04_05_2024__13:37:15
* UniqueID: 295b8a71-90d7-49a8-b3a8-5cf97757f904
* Gherkinscenario name: Gherkinscenario 2: Money withdraw from bank account
* LLM-temperature: 1.3
## Gherkin Input
```
Given: UserA bank account is in credit
When: UserA attempts to withdraw an amount less than UserA card's limit
Then: the withdrawal should be complete without errors or warnings
```
## Input prompt, technique: Chain-of-Thought
Learn from the following example, the output information for the CNL Rimay is:
This part is the start of the example you have to learn from:
The input information is as follows:
Acceptance Criteria 1 in Gherkin, input:
Gherkin:
Scenario: Create an Order
Given an Order of type Subscription_Order does not exist in OI of type Order_Issuer
When OI Create Order,
Then Order exists in OI
And the property settlement_method of Order is equal to FOP
Rimay output 1:
When the Order_Issuer (OI ) executes Create a Subscription_Order, if the Order does not exist, then the Order is created and its settlement_method is set to FOP.
Rimay uses the following structure:
When the? Actor <Action> (every "Frequency")? ,|then the? Actor must <Action> (every "Text")?.
Ending for example
Translate the following input acceptance criteria Gherkin, according to the example above, to Rimay:
### Input Data
Use the following Gherkin acceptance criteria input:
```
Given: UserA bank account is in credit
When: UserA attempts to withdraw an amount less than UserA card's limit
Then: the withdrawal should be complete without errors or warnings
```
### Question:
Can you translate the input above into the specified language Rimay CNL?
### Output indicator
Output only the translated Rimay CNL text and nothing else!
## Rimay Output
```
Given UserA's bank account is in credit, When UserA attempts a withdrawal amount less than their card limit,
Then the withdrawal should be completed without any errors or warnings.
```
### Researcher score
Researcher_score = -10
## DSL-Rimay Verification
```
ERROR:no viable alternative at input 'UserA' (line : 3 column : 7)
ERROR:mismatched character '<EOF>' expecting ''' (line : 3 column : 12)
ERROR:Couldn't resolve reference to Actor 'Given'. (line : 3 column : 1)
```
### DSL-Rimay Score
DSL_Score: 86
Progressbar: ![86%](https://progress-bar.dev/86)
## Rimay Paska Verification
```
File generated_rimay.csv.csv
Smells found in this document:
Non-atomic requirement: 0
Incomplete requirement: 0
Incorrect order requirement: 0
Coordination ambiguity: 0
Not requirement: ,0
Incomplete condition: 0
Incomplete system response: 0
Passive voice: 1
Not precise verb: 0
Number Req with smells: 1
```
### Paska Score
PASKA_Score: 95
Progressbar: ![95%](https://progress-bar.dev/95)
### Final Score
Final_score = 80.5
Progressbar: ![80.5%](https://progress-bar.dev/80.5)
output_dataset/Few-shot-learning/Scenario 10: Logging out the user doesn't log out my admin account_Incorrect___04_02_2024__21:32:58__70.0_score.md → output_dataset/Few-shot-learning_0.2temp/Gherkinscenario 10: Logging out the user doesn't log out my admin account_Incorrect___04_02_2024__21:32:58__70.0_score.md
View File
output_dataset/Few-shot-learning/Scenario 10: Logging out the user doesn't log out my admin account__04_02_2024__21:31:57__87.0_score.md → output_dataset/Few-shot-learning_0.2temp/Gherkinscenario 10: Logging out the user doesn't log out my admin account__04_02_2024__21:31:57__87.0_score.md
View File
output_dataset/Few-shot-learning_0.2temp/Gherkinscenario 11: Seeing disabled code field while editing country_Incorrect___04_05_2024__13:55:01__70.5_score.md
+128
View File
@@ -0,0 +1,128 @@
# LLM Experiment Information
* Prefix: Few-shot-learning
* Datetime: 04_05_2024__13:55:01
* UniqueID: 0c505e26-6ebc-469f-a352-cd5fe3c73458
* Gherkinscenario name: Gherkinscenario 11: Seeing disabled code field while editing country_Incorrect_
* LLM-temperature: 0.2
## Gherkin Input
```
Given: SystemA has country "United States"
When: UserA wants to edit this country
Then: the code field should be disabled
```
## Input prompt, technique: Few-Shot-Learning
### Context
We are translating Gherkin acceptance criteria into Rimay CNL.
For this translation use the following information:
$ACTOR is for example a thing or a person.
$MODAL_VERB is for the following word: should or can
$TRIGGER is for example a trigger action, nothing what a system does.
$WHILE_STRUCTUREis the word 'When' followed by the Gherkin When condition
$WHEN_STRUCTURE is the word 'While' followed by the Gherkin Given condition
$SYSTEM_RESPONSE is a combination of the Actors and the Gherkin Then condition.
$ACTOR, $MODAL_VERB and a $SYSTEM_RESPONSE is mandatory in all Rimay requirements.
Replace the variables with the right text according the input acceptance criteria.
In the following order:
Rimay CNL definition: $SYSTEM_RESPONSE $MODAL_VERB $WHILE_STRUCTURE $ACTOR $WHEN_STRUCTURE
With the usage of the following extra information:
Actors: actor SystemA, actor SystemB, actor SystemC, actor UserX, actor UserA, actor UserC
### Input Data
Use the following Gherkin acceptance criteria input:
```
Given: SystemA has country "United States"
When: UserA wants to edit this country
Then: the code field should be disabled
```
### Question:
Can you translate the input above into the specified language Rimay CNL?
### Output indicator
Output only the translated Rimay CNL text and nothing else!
## Rimay Output
```
UserA wants to edit this country should When SystemA has country "United States" UserA
```
### Researcher score
Researcher_score = -10
## DSL-Rimay Verification
```
ERROR:no viable alternative at input 'wants' (line : 3 column : 7)
ERROR:missing '.' at 'edit' (line : 3 column : 16)
ERROR:no viable alternative at input 'this' (line : 3 column : 21)
ERROR:missing '.' at 'country' (line : 3 column : 26)
ERROR:no viable alternative at input 'should' (line : 3 column : 34)
ERROR:missing '.' at 'When' (line : 3 column : 42)
ERROR:no viable alternative at input 'has' (line : 3 column : 55)
ERROR:missing ',' at 'country' (line : 3 column : 59)
ERROR:no viable alternative at input '"United States"' (line : 3 column : 67)
ERROR:no viable alternative at input 'UserA' (line : 3 column : 84)
ERROR:Couldn't resolve reference to Actor 'edit'. (line : 3 column : 16)
ERROR:Couldn't resolve reference to Actor 'country'. (line : 3 column : 26)
ERROR:Couldn't resolve reference to Actor 'country'. (line : 3 column : 59)
```
### DSL-Rimay Score
DSL_Score: 81
Progressbar: ![81%](https://progress-bar.dev/81)
## Rimay Paska Verification
```
File generated_rimay.csv.csv
Smells found in this document:
Non-atomic requirement: 0
Incomplete requirement: 0
Incorrect order requirement: 1
Coordination ambiguity: 0
Not requirement: ,0
Incomplete condition: 0
Incomplete system response: 1
Passive voice: 0
Not precise verb: 0
Number Req with smells: 1
```
### Paska Score
PASKA_Score: 80
Progressbar: ![80%](https://progress-bar.dev/80)
### Final Score
Final_score = 70.5
Progressbar: ![70.5%](https://progress-bar.dev/70.5)
output_dataset/Few-shot-learning_0.2temp/Gherkinscenario 11: Seeing disabled code field while editing country__04_05_2024__13:53:52__98.0_score.md
+137
View File
@@ -0,0 +1,137 @@
# LLM Experiment Information
* Prefix: Few-shot-learning
* Datetime: 04_05_2024__13:53:52
* UniqueID: cdb7956c-52eb-4520-befb-f97b3e47e84e
* Gherkinscenario name: Gherkinscenario 11: Seeing disabled code field while editing country
* LLM-temperature: 0.2
## Gherkin Input
```
Given: SystemA has country "United States"
When: UserA wants to edit this country
Then: the code field should be disabled
```
## Input prompt, technique: Few-Shot-Learning
### Context
We are translating Gherkin acceptance criteria into Rimay CNL.
For this translation use the following information:
$ACTOR is for example a thing or a person.
$MODAL_VERB is for the following word: must or shall.
$TRIGGER is for example a trigger action, something what a system does.
$WHILE_STRUCTURE is the word 'While' followed by the Gherkin Given condition
$WHEN_STRUCTURE is the word 'When' followed by the Gherkin When condition
$SYSTEM_RESPONSE is a combination of the Actors and the Gherkin Then condition.
$ACTOR, $MODAL_VERB and a $SYSTEM_RESPONSE is mandatory in all Rimay requirements.
Replace the variables with the right text according the input acceptance criteria.
In the following order:
Rimay CNL definition: $WHILE_STRUCTURE $WHEN_STRUCTURE $ACTOR $MODAL_VERB $SYSTEM_RESPONSE
The following example is a valid Rimay CNL:
When SystemB receives an "email alert" from SystemA, SystemB must send an Instruction to SystemC.
With the usage of the following extra information:
Actors: actor SystemA, actor SystemB, actor SystemC, actor UserX, actor UserA, actor UserC
### Input Data
Use the following Gherkin acceptance criteria input:
```
Given: SystemA has country "United States"
When: UserA wants to edit this country
Then: the code field should be disabled
```
### Question:
Can you translate the input above into the specified language Rimay CNL?
### Output indicator
Output only the translated Rimay CNL text and nothing else!
## Rimay Output
```
While SystemA has country "United States" When UserA wants to edit this country UserA must the code field should be disabled
```
### Researcher score
Researcher_score = 10
## DSL-Rimay Verification
```
ERROR:no viable alternative at input 'to' (line : 3 column : 62)
ERROR:missing ',' at 'edit' (line : 3 column : 65)
ERROR:no viable alternative at input 'this' (line : 3 column : 70)
ERROR:missing '.' at 'country' (line : 3 column : 75)
ERROR:no viable alternative at input 'UserA' (line : 3 column : 85)
ERROR:missing '.' at 'the' (line : 3 column : 100)
ERROR:no viable alternative at input 'field' (line : 3 column : 109)
ERROR:missing '.' at 'should' (line : 3 column : 115)
ERROR:no viable alternative at input 'be' (line : 3 column : 122)
ERROR:missing '.' at 'disabled' (line : 3 column : 125)
ERROR:Couldn't resolve reference to Quantifier 'country'. (line : 3 column : 19)
ERROR:Couldn't resolve reference to Actor 'wants'. (line : 3 column : 56)
ERROR:Couldn't resolve reference to Quantifier 'UserA'. (line : 3 column : 50)
ERROR:Couldn't resolve reference to Actor 'edit'. (line : 3 column : 65)
ERROR:Couldn't resolve reference to Actor 'country'. (line : 3 column : 75)
ERROR:Couldn't resolve reference to Actor 'code'. (line : 3 column : 104)
ERROR:Couldn't resolve reference to Actor 'should'. (line : 3 column : 115)
ERROR:Couldn't resolve reference to Actor 'disabled'. (line : 3 column : 125)
```
### DSL-Rimay Score
DSL_Score: 76
Progressbar: ![76%](https://progress-bar.dev/76)
## Rimay Paska Verification
```
File generated_rimay.csv.csv
Smells found in this document:
Non-atomic requirement: 0
Incomplete requirement: 0
Incorrect order requirement: 0
Coordination ambiguity: 0
Not requirement: ,0
Incomplete condition: 0
Incomplete system response: 0
Passive voice: 0
Not precise verb: 0
Number Req with smells: 0
```
### Paska Score
PASKA_Score: 100
Progressbar: ![100%](https://progress-bar.dev/100)
### Final Score
Final_score = 98.0
Progressbar: ![98.0%](https://progress-bar.dev/98.0)
output_dataset/Few-shot-learning_0.2temp/Gherkinscenario 12: Changing the currency of my cart_Incorrect___04_05_2024__14:01:05__69.0_score.md
+136
View File
@@ -0,0 +1,136 @@
# LLM Experiment Information
* Prefix: Few-shot-learning
* Datetime: 04_05_2024__14:01:05
* UniqueID: 21c7e0db-9dce-49cf-9d09-ee40678cfe1d
* Gherkinscenario name: Gherkinscenario 12: Changing the currency of my cart_Incorrect_
* LLM-temperature: 0.2
## Gherkin Input
```
Given: UserA has product "The Pug Mug" in the cart
When: UserA switches to the "EUR" currency
Then: the grand total value should be "€6.82"
```
## Input prompt, technique: Few-Shot-Learning
### Context
We are translating Gherkin acceptance criteria into Rimay CNL.
For this translation use the following information:
$ACTOR is for example a thing or a person.
$MODAL_VERB is for the following word: should or can
$TRIGGER is for example a trigger action, nothing what a system does.
$WHILE_STRUCTUREis the word 'When' followed by the Gherkin When condition
$WHEN_STRUCTURE is the word 'While' followed by the Gherkin Given condition
$SYSTEM_RESPONSE is a combination of the Actors and the Gherkin Then condition.
$ACTOR, $MODAL_VERB and a $SYSTEM_RESPONSE is mandatory in all Rimay requirements.
Replace the variables with the right text according the input acceptance criteria.
In the following order:
Rimay CNL definition: $SYSTEM_RESPONSE $MODAL_VERB $WHILE_STRUCTURE $ACTOR $WHEN_STRUCTURE
With the usage of the following extra information:
Actors: actor SystemA, actor SystemB, actor SystemC, actor UserX, actor UserA, actor UserC
### Input Data
Use the following Gherkin acceptance criteria input:
```
Given: UserA has product "The Pug Mug" in the cart
When: UserA switches to the "EUR" currency
Then: the grand total value should be "€6.82"
```
### Question:
Can you translate the input above into the specified language Rimay CNL?
### Output indicator
Output only the translated Rimay CNL text and nothing else!
## Rimay Output
```
the grand total value should be "€6.82" should While UserA switches to the "EUR" currency SystemA When UserA has product "The Pug Mug" in the cart
```
### Researcher score
Researcher_score = -10
## DSL-Rimay Verification
```
ERROR:no viable alternative at input 'total' (line : 3 column : 11)
ERROR:missing '.' at 'should' (line : 3 column : 23)
ERROR:no viable alternative at input 'be' (line : 3 column : 30)
ERROR:missing '.' at 'should' (line : 3 column : 41)
ERROR:no viable alternative at input 'While' (line : 3 column : 51)
ERROR:missing '.' at 'UserA' (line : 3 column : 57)
ERROR:no viable alternative at input 'switches' (line : 3 column : 63)
ERROR:missing '.' at 'the' (line : 3 column : 75)
ERROR:extraneous input '"EUR"' expecting RULE_ID (line : 3 column : 79)
ERROR:no viable alternative at input 'SystemA' (line : 3 column : 97)
ERROR:no viable alternative at input 'has' (line : 3 column : 119)
ERROR:missing ',' at 'product' (line : 3 column : 123)
ERROR:no viable alternative at input '"The Pug Mug"' (line : 3 column : 131)
ERROR:no viable alternative at input 'the' (line : 3 column : 148)
ERROR:no viable alternative at input '<EOF>' (line : 3 column : 152)
ERROR:Couldn't resolve reference to Actor 'grand'. (line : 3 column : 5)
ERROR:Couldn't resolve reference to Actor 'should'. (line : 3 column : 23)
ERROR:Couldn't resolve reference to Actor 'should'. (line : 3 column : 41)
ERROR:Couldn't resolve reference to Actor 'currency'. (line : 3 column : 85)
ERROR:Couldn't resolve reference to Actor 'product'. (line : 3 column : 123)
ERROR:Couldn't resolve reference to Actor 'cart'. (line : 3 column : 152)
```
### DSL-Rimay Score
DSL_Score: 63
Progressbar: ![63%](https://progress-bar.dev/63)
## Rimay Paska Verification
```
File generated_rimay.csv.csv
Smells found in this document:
Non-atomic requirement: 0
Incomplete requirement: 0
Incorrect order requirement: 1
Coordination ambiguity: 0
Not requirement: ,0
Incomplete condition: 0
Incomplete system response: 0
Passive voice: 0
Not precise verb: 0
Number Req with smells: 1
```
### Paska Score
PASKA_Score: 95
Progressbar: ![95%](https://progress-bar.dev/95)
### Final Score
Final_score = 69.0
Progressbar: ![69.0%](https://progress-bar.dev/69.0)
output_dataset/Few-shot-learning_0.2temp/Gherkinscenario 12: Changing the currency of my cart__04_05_2024__13:55:49__86.0_score.md
+136
View File
@@ -0,0 +1,136 @@
# LLM Experiment Information
* Prefix: Few-shot-learning
* Datetime: 04_05_2024__13:55:49
* UniqueID: f513b7fb-f65f-4ced-b274-1bbb3b68df85
* Gherkinscenario name: Gherkinscenario 12: Changing the currency of my cart
* LLM-temperature: 0.2
## Gherkin Input
```
Given: UserA has product "The Pug Mug" in the cart
When: UserA switches to the "EUR" currency
Then: the grand total value should be "€6.82"
```
## Input prompt, technique: Few-Shot-Learning
### Context
We are translating Gherkin acceptance criteria into Rimay CNL.
For this translation use the following information:
$ACTOR is for example a thing or a person.
$MODAL_VERB is for the following word: must or shall.
$TRIGGER is for example a trigger action, something what a system does.
$WHILE_STRUCTURE is the word 'While' followed by the Gherkin Given condition
$WHEN_STRUCTURE is the word 'When' followed by the Gherkin When condition
$SYSTEM_RESPONSE is a combination of the Actors and the Gherkin Then condition.
$ACTOR, $MODAL_VERB and a $SYSTEM_RESPONSE is mandatory in all Rimay requirements.
Replace the variables with the right text according the input acceptance criteria.
In the following order:
Rimay CNL definition: $WHILE_STRUCTURE $WHEN_STRUCTURE $ACTOR $MODAL_VERB $SYSTEM_RESPONSE
The following example is a valid Rimay CNL:
When SystemB receives an "email alert" from SystemA, SystemB must send an Instruction to SystemC.
With the usage of the following extra information:
Actors: actor SystemA, actor SystemB, actor SystemC, actor UserX, actor UserA, actor UserC
### Input Data
Use the following Gherkin acceptance criteria input:
```
Given: UserA has product "The Pug Mug" in the cart
When: UserA switches to the "EUR" currency
Then: the grand total value should be "€6.82"
```
### Question:
Can you translate the input above into the specified language Rimay CNL?
### Output indicator
Output only the translated Rimay CNL text and nothing else!
## Rimay Output
```
While UserA has product "The Pug Mug" in the cart When UserA switches to the "EUR" currency UserA must the grand total value should be "€6.82"
```
### Researcher score
Researcher_score = 15
## DSL-Rimay Verification
```
ERROR:mismatched input 'in' expecting ',' (line : 3 column : 39)
ERROR:missing '.' at 'the' (line : 3 column : 42)
ERROR:no viable alternative at input 'When' (line : 3 column : 53)
ERROR:missing '.' at 'UserA' (line : 3 column : 58)
ERROR:no viable alternative at input 'switches' (line : 3 column : 64)
ERROR:missing '.' at 'the' (line : 3 column : 76)
ERROR:extraneous input '"EUR"' expecting RULE_ID (line : 3 column : 80)
ERROR:no viable alternative at input 'UserA' (line : 3 column : 97)
ERROR:no viable alternative at input 'total' (line : 3 column : 122)
ERROR:missing '.' at 'should' (line : 3 column : 134)
ERROR:no viable alternative at input 'be' (line : 3 column : 141)
ERROR:mismatched input '<EOF>' expecting '.' (line : 3 column : 144)
ERROR:Couldn't resolve reference to Quantifier 'product'. (line : 3 column : 17)
ERROR:Couldn't resolve reference to Actor 'cart'. (line : 3 column : 46)
ERROR:Couldn't resolve reference to Actor 'currency'. (line : 3 column : 86)
ERROR:Couldn't resolve reference to Actor 'grand'. (line : 3 column : 116)
ERROR:Couldn't resolve reference to Actor 'should'. (line : 3 column : 134)
```
### DSL-Rimay Score
DSL_Score: 67
Progressbar: ![67%](https://progress-bar.dev/67)
## Rimay Paska Verification
```
File generated_rimay.csv.csv
Smells found in this document:
Non-atomic requirement: 0
Incomplete requirement: 1
Incorrect order requirement: 0
Coordination ambiguity: 0
Not requirement: ,0
Incomplete condition: 0
Incomplete system response: 0
Passive voice: 0
Not precise verb: 0
Number Req with smells: 1
```
### Paska Score
PASKA_Score: 75
Progressbar: ![75%](https://progress-bar.dev/75)
### Final Score
Final_score = 86.0
Progressbar: ![86.0%](https://progress-bar.dev/86.0)
output_dataset/Few-shot-learning_0.2temp/Gherkinscenario 13: Viewing information about empty cart after channel switching_Incorrect___04_05_2024__14:03:29__67.0_score.md
+140
View File
@@ -0,0 +1,140 @@
# LLM Experiment Information
* Prefix: Few-shot-learning
* Datetime: 04_05_2024__14:03:29
* UniqueID: 11cc7fb8-25d5-4d5c-b7f5-ed6b9c898220
* Gherkinscenario name: Gherkinscenario 13: Viewing information about empty cart after channel switching_Incorrect_
* LLM-temperature: 0.2
## Gherkin Input
```
Given: UserA simplifies his current channel to "Poland"
When: UserA changes current channel to "France"
Then: UserA carts should be empty
```
## Input prompt, technique: Few-Shot-Learning
### Context
We are translating Gherkin acceptance criteria into Rimay CNL.
For this translation use the following information:
$ACTOR is for example a thing or a person.
$MODAL_VERB is for the following word: should or can
$TRIGGER is for example a trigger action, nothing what a system does.
$WHILE_STRUCTUREis the word 'When' followed by the Gherkin When condition
$WHEN_STRUCTURE is the word 'While' followed by the Gherkin Given condition
$SYSTEM_RESPONSE is a combination of the Actors and the Gherkin Then condition.
$ACTOR, $MODAL_VERB and a $SYSTEM_RESPONSE is mandatory in all Rimay requirements.
Replace the variables with the right text according the input acceptance criteria.
In the following order:
Rimay CNL definition: $SYSTEM_RESPONSE $MODAL_VERB $WHILE_STRUCTURE $ACTOR $WHEN_STRUCTURE
With the usage of the following extra information:
Actors: actor SystemA, actor SystemB, actor SystemC, actor UserX, actor UserA, actor UserC
### Input Data
Use the following Gherkin acceptance criteria input:
```
Given: UserA simplifies his current channel to "Poland"
When: UserA changes current channel to "France"
Then: UserA carts should be empty
```
### Question:
Can you translate the input above into the specified language Rimay CNL?
### Output indicator
Output only the translated Rimay CNL text and nothing else!
## Rimay Output
```
UserA carts should be empty should UserA can While UserA changes current channel to "France" Given UserA simplifies his current channel to "Poland"
```
### Researcher score
Researcher_score = -10
## DSL-Rimay Verification
```
ERROR:no viable alternative at input 'carts' (line : 3 column : 7)
ERROR:missing '.' at 'should' (line : 3 column : 13)
ERROR:no viable alternative at input 'be' (line : 3 column : 20)
ERROR:missing '.' at 'empty' (line : 3 column : 23)
ERROR:no viable alternative at input 'should' (line : 3 column : 29)
ERROR:missing '.' at 'UserA' (line : 3 column : 36)
ERROR:no viable alternative at input 'can' (line : 3 column : 42)
ERROR:missing '.' at 'While' (line : 3 column : 46)
ERROR:no viable alternative at input 'changes' (line : 3 column : 52)
ERROR:no viable alternative at input 'changes' (line : 3 column : 58)
ERROR:missing 'accept' at '"France"' (line : 3 column : 85)
ERROR:no viable alternative at input 'Given' (line : 3 column : 94)
ERROR:no viable alternative at input 'UserA' (line : 3 column : 100)
ERROR:missing '.' at 'simplifies' (line : 3 column : 106)
ERROR:no viable alternative at input 'his' (line : 3 column : 117)
ERROR:missing '.' at 'current' (line : 3 column : 121)
ERROR:no viable alternative at input 'channel' (line : 3 column : 129)
ERROR:mismatched input '<EOF>' expecting '.' (line : 3 column : 140)
ERROR:Couldn't resolve reference to Actor 'should'. (line : 3 column : 13)
ERROR:Couldn't resolve reference to Actor 'empty'. (line : 3 column : 23)
ERROR:Couldn't resolve reference to Quantifier 'current'. (line : 3 column : 66)
ERROR:Couldn't resolve reference to PropertyId 'channel'. (line : 3 column : 74)
ERROR:Couldn't resolve reference to Actor 'Given'. (line : 3 column : 94)
ERROR:Couldn't resolve reference to Actor 'simplifies'. (line : 3 column : 106)
ERROR:Couldn't resolve reference to Actor 'current'. (line : 3 column : 121)
```
### DSL-Rimay Score
DSL_Score: 59
Progressbar: ![59%](https://progress-bar.dev/59)
## Rimay Paska Verification
```
File generated_rimay.csv.csv
Smells found in this document:
Non-atomic requirement: 0
Incomplete requirement: 0
Incorrect order requirement: 1
Coordination ambiguity: 0
Not requirement: ,0
Incomplete condition: 0
Incomplete system response: 0
Passive voice: 0
Not precise verb: 0
Number Req with smells: 1
```
### Paska Score
PASKA_Score: 95
Progressbar: ![95%](https://progress-bar.dev/95)
### Final Score
Final_score = 67.0
Progressbar: ![67.0%](https://progress-bar.dev/67.0)
output_dataset/Few-shot-learning_0.2temp/Gherkinscenario 13: Viewing information about empty cart after channel switching__04_05_2024__14:02:05__100_score.md
+136
View File
@@ -0,0 +1,136 @@
# LLM Experiment Information
* Prefix: Few-shot-learning
* Datetime: 04_05_2024__14:02:05
* UniqueID: a445ba8b-d812-431e-b4ae-857077655244
* Gherkinscenario name: Gherkinscenario 13: Viewing information about empty cart after channel switching
* LLM-temperature: 0.2
## Gherkin Input
```
Given: UserA simplifies his current channel to "Poland"
When: UserA changes current channel to "France"
Then: UserA carts should be empty
```
## Input prompt, technique: Few-Shot-Learning
### Context
We are translating Gherkin acceptance criteria into Rimay CNL.
For this translation use the following information:
$ACTOR is for example a thing or a person.
$MODAL_VERB is for the following word: must or shall.
$TRIGGER is for example a trigger action, something what a system does.
$WHILE_STRUCTURE is the word 'While' followed by the Gherkin Given condition
$WHEN_STRUCTURE is the word 'When' followed by the Gherkin When condition
$SYSTEM_RESPONSE is a combination of the Actors and the Gherkin Then condition.
$ACTOR, $MODAL_VERB and a $SYSTEM_RESPONSE is mandatory in all Rimay requirements.
Replace the variables with the right text according the input acceptance criteria.
In the following order:
Rimay CNL definition: $WHILE_STRUCTURE $WHEN_STRUCTURE $ACTOR $MODAL_VERB $SYSTEM_RESPONSE
The following example is a valid Rimay CNL:
When SystemB receives an "email alert" from SystemA, SystemB must send an Instruction to SystemC.
With the usage of the following extra information:
Actors: actor SystemA, actor SystemB, actor SystemC, actor UserX, actor UserA, actor UserC
### Input Data
Use the following Gherkin acceptance criteria input:
```
Given: UserA simplifies his current channel to "Poland"
When: UserA changes current channel to "France"
Then: UserA carts should be empty
```
### Question:
Can you translate the input above into the specified language Rimay CNL?
### Output indicator
Output only the translated Rimay CNL text and nothing else!
## Rimay Output
```
While UserA simplifies his current channel to "Poland" When UserA changes current channel to "France" UserA must have empty carts
```
### Researcher score
Researcher_score = 15
## DSL-Rimay Verification
```
ERROR:no viable alternative at input 'his' (line : 3 column : 24)
ERROR:no viable alternative at input 'current' (line : 3 column : 28)
ERROR:missing '.' at 'channel' (line : 3 column : 36)
ERROR:no viable alternative at input 'to' (line : 3 column : 44)
ERROR:no viable alternative at input 'When' (line : 3 column : 58)
ERROR:missing 'accept' at '"France"' (line : 3 column : 96)
ERROR:no viable alternative at input 'UserA' (line : 3 column : 107)
ERROR:no viable alternative at input 'have' (line : 3 column : 118)
ERROR:missing '.' at 'empty' (line : 3 column : 123)
ERROR:no viable alternative at input 'carts' (line : 3 column : 129)
ERROR:Couldn't resolve reference to ActorOrClassOrProperty 'simplifies'. (line : 3 column : 13)
ERROR:Couldn't resolve reference to Quantifier 'UserA'. (line : 3 column : 7)
ERROR:Couldn't resolve reference to Actor 'his'. (line : 3 column : 24)
ERROR:Couldn't resolve reference to Actor 'channel'. (line : 3 column : 36)
ERROR:Couldn't resolve reference to Quantifier 'current'. (line : 3 column : 77)
ERROR:Couldn't resolve reference to PropertyId 'channel'. (line : 3 column : 85)
ERROR:Couldn't resolve reference to Actor 'empty'. (line : 3 column : 123)
```
### DSL-Rimay Score
DSL_Score: 77
Progressbar: ![77%](https://progress-bar.dev/77)
## Rimay Paska Verification
```
File generated_rimay.csv.csv
Smells found in this document:
Non-atomic requirement: 0
Incomplete requirement: 0
Incorrect order requirement: 0
Coordination ambiguity: 0
Not requirement: ,0
Incomplete condition: 0
Incomplete system response: 0
Passive voice: 0
Not precise verb: 0
Number Req with smells: 0
```
### Paska Score
PASKA_Score: 100
Progressbar: ![100%](https://progress-bar.dev/100)
### Final Score
Final_score = 100
Progressbar: ![100%](https://progress-bar.dev/100)
output_dataset/Few-shot-learning_0.2temp/Gherkinscenario 14: Displaying default shop homepage_Incorrect___04_05_2024__14:05:40__65.5_score.md
+133
View File
@@ -0,0 +1,133 @@
# LLM Experiment Information
* Prefix: Few-shot-learning
* Datetime: 04_05_2024__14:05:40
* UniqueID: b949b197-c744-4d6a-a908-a9d6460b15d6
* Gherkinscenario name: Gherkinscenario 14: Displaying default shop homepage_Incorrect_
* LLM-temperature: 0.2
## Gherkin Input
```
Given: channel "United States" does not use any theme
When: UserA visits this channel's homepage
Then: UserA should not see a homepage from "maverick/meerkat" theme
```
## Input prompt, technique: Few-Shot-Learning
### Context
We are translating Gherkin acceptance criteria into Rimay CNL.
For this translation use the following information:
$ACTOR is for example a thing or a person.
$MODAL_VERB is for the following word: should or can
$TRIGGER is for example a trigger action, nothing what a system does.
$WHILE_STRUCTUREis the word 'When' followed by the Gherkin When condition
$WHEN_STRUCTURE is the word 'While' followed by the Gherkin Given condition
$SYSTEM_RESPONSE is a combination of the Actors and the Gherkin Then condition.
$ACTOR, $MODAL_VERB and a $SYSTEM_RESPONSE is mandatory in all Rimay requirements.
Replace the variables with the right text according the input acceptance criteria.
In the following order:
Rimay CNL definition: $SYSTEM_RESPONSE $MODAL_VERB $WHILE_STRUCTURE $ACTOR $WHEN_STRUCTURE
With the usage of the following extra information:
Actors: actor SystemA, actor SystemB, actor SystemC, actor UserX, actor UserA, actor UserC
### Input Data
Use the following Gherkin acceptance criteria input:
```
Given: channel "United States" does not use any theme
When: UserA visits this channel's homepage
Then: UserA should not see a homepage from "maverick/meerkat" theme
```
### Question:
Can you translate the input above into the specified language Rimay CNL?
### Output indicator
Output only the translated Rimay CNL text and nothing else!
## Rimay Output
```
UserA should not see a homepage from "maverick/meerkat" theme should SystemA While UserA visits this channel's homepage
```
### Researcher score
Researcher_score = -10
## DSL-Rimay Verification
```
ERROR:no viable alternative at input 'should' (line : 3 column : 7)
ERROR:missing '.' at 'see' (line : 3 column : 18)
ERROR:no viable alternative at input 'a' (line : 3 column : 22)
ERROR:missing '.' at 'homepage' (line : 3 column : 24)
ERROR:no viable alternative at input 'from' (line : 3 column : 33)
ERROR:no viable alternative at input 'theme' (line : 3 column : 57)
ERROR:no viable alternative at input 'should' (line : 3 column : 63)
ERROR:missing '.' at 'SystemA' (line : 3 column : 70)
ERROR:no viable alternative at input 'While' (line : 3 column : 78)
ERROR:missing '.' at 'UserA' (line : 3 column : 84)
ERROR:no viable alternative at input 'visits' (line : 3 column : 90)
ERROR:missing '.' at 'this' (line : 3 column : 97)
ERROR:no viable alternative at input 'channel' (line : 3 column : 102)
ERROR:mismatched character '<EOF>' expecting ''' (line : 3 column : 109)
ERROR:Couldn't resolve reference to Actor 'see'. (line : 3 column : 18)
ERROR:Couldn't resolve reference to Actor 'homepage'. (line : 3 column : 24)
ERROR:Couldn't resolve reference to Actor 'theme'. (line : 3 column : 57)
ERROR:Couldn't resolve reference to Actor 'this'. (line : 3 column : 97)
```
### DSL-Rimay Score
DSL_Score: 66
Progressbar: ![66%](https://progress-bar.dev/66)
## Rimay Paska Verification
```
File generated_rimay.csv.csv
Smells found in this document:
Non-atomic requirement: 0
Incomplete requirement: 0
Incorrect order requirement: 1
Coordination ambiguity: 0
Not requirement: ,0
Incomplete condition: 1
Incomplete system response: 0
Passive voice: 0
Not precise verb: 0
Number Req with smells: 1
```
### Paska Score
PASKA_Score: 85
Progressbar: ![85%](https://progress-bar.dev/85)
### Final Score
Final_score = 65.5
Progressbar: ![65.5%](https://progress-bar.dev/65.5)
output_dataset/Few-shot-learning_0.2temp/Gherkinscenario 14: Displaying default shop homepage__04_05_2024__14:04:26__100_score.md
+131
View File
@@ -0,0 +1,131 @@
# LLM Experiment Information
* Prefix: Few-shot-learning
* Datetime: 04_05_2024__14:04:26
* UniqueID: 07e0f9dc-aff6-438c-a75d-52850fe2471d
* Gherkinscenario name: Gherkinscenario 14: Displaying default shop homepage
* LLM-temperature: 0.2
## Gherkin Input
```
Given: channel "United States" does not use any theme
When: UserA visits this channel's homepage
Then: UserA should not see a homepage from "maverick/meerkat" theme
```
## Input prompt, technique: Few-Shot-Learning
### Context
We are translating Gherkin acceptance criteria into Rimay CNL.
For this translation use the following information:
$ACTOR is for example a thing or a person.
$MODAL_VERB is for the following word: must or shall.
$TRIGGER is for example a trigger action, something what a system does.
$WHILE_STRUCTURE is the word 'While' followed by the Gherkin Given condition
$WHEN_STRUCTURE is the word 'When' followed by the Gherkin When condition
$SYSTEM_RESPONSE is a combination of the Actors and the Gherkin Then condition.
$ACTOR, $MODAL_VERB and a $SYSTEM_RESPONSE is mandatory in all Rimay requirements.
Replace the variables with the right text according the input acceptance criteria.
In the following order:
Rimay CNL definition: $WHILE_STRUCTURE $WHEN_STRUCTURE $ACTOR $MODAL_VERB $SYSTEM_RESPONSE
The following example is a valid Rimay CNL:
When SystemB receives an "email alert" from SystemA, SystemB must send an Instruction to SystemC.
With the usage of the following extra information:
Actors: actor SystemA, actor SystemB, actor SystemC, actor UserX, actor UserA, actor UserC
### Input Data
Use the following Gherkin acceptance criteria input:
```
Given: channel "United States" does not use any theme
When: UserA visits this channel's homepage
Then: UserA should not see a homepage from "maverick/meerkat" theme
```
### Question:
Can you translate the input above into the specified language Rimay CNL?
### Output indicator
Output only the translated Rimay CNL text and nothing else!
## Rimay Output
```
While channel "United States" does not use any theme When UserA visits this channel's homepage UserA must not UserA should not see a homepage from "maverick/meerkat" theme
```
### Researcher score
Researcher_score = 15
## DSL-Rimay Verification
```
ERROR:no viable alternative at input 'use' (line : 3 column : 7)
ERROR:no viable alternative at input '"United States"' (line : 3 column : 15)
ERROR:missing '.' at 'When' (line : 3 column : 56)
ERROR:no viable alternative at input 'this' (line : 3 column : 74)
ERROR:no viable alternative at input 'channel' (line : 3 column : 79)
ERROR:mismatched character '<EOF>' expecting ''' (line : 3 column : 86)
ERROR:Couldn't resolve reference to Actor 'channel'. (line : 3 column : 7)
ERROR:Couldn't resolve reference to ClassOrLabelOrProperty 'theme'. (line : 3 column : 48)
ERROR:Couldn't resolve reference to Quantifier 'any'. (line : 3 column : 44)
ERROR:Couldn't resolve reference to Actor 'visits'. (line : 3 column : 67)
ERROR:Couldn't resolve reference to Quantifier 'UserA'. (line : 3 column : 61)
ERROR:Couldn't resolve reference to Actor 'this'. (line : 3 column : 74)
```
### DSL-Rimay Score
DSL_Score: 72
Progressbar: ![72%](https://progress-bar.dev/72)
## Rimay Paska Verification
```
File generated_rimay.csv.csv
Smells found in this document:
Non-atomic requirement: 0
Incomplete requirement: 0
Incorrect order requirement: 0
Coordination ambiguity: 0
Not requirement: ,0
Incomplete condition: 0
Incomplete system response: 0
Passive voice: 0
Not precise verb: 0
Number Req with smells: 0
```
### Paska Score
PASKA_Score: 100
Progressbar: ![100%](https://progress-bar.dev/100)
### Final Score
Final_score = 100
Progressbar: ![100%](https://progress-bar.dev/100)
output_dataset/Few-shot-learning_0.2temp/Gherkinscenario 15: Displaying themed shop homepage_Incorrect___04_05_2024__14:07:45__71.0_score.md
+132
View File
@@ -0,0 +1,132 @@
# LLM Experiment Information
* Prefix: Few-shot-learning
* Datetime: 04_05_2024__14:07:45
* UniqueID: d43b5c9f-8228-4f9f-b3b0-9375d2e9530b
* Gherkinscenario name: Gherkinscenario 15: Displaying themed shop homepage_Incorrect_
* LLM-temperature: 0.2
## Gherkin Input
```
Given: channel "United States" uses "maverick/meerkat" theme
When: UserA visit this channel's homepage
Then: UserA should see a homepage from that theme
```
## Input prompt, technique: Few-Shot-Learning
### Context
We are translating Gherkin acceptance criteria into Rimay CNL.
For this translation use the following information:
$ACTOR is for example a thing or a person.
$MODAL_VERB is for the following word: should or can
$TRIGGER is for example a trigger action, nothing what a system does.
$WHILE_STRUCTUREis the word 'When' followed by the Gherkin When condition
$WHEN_STRUCTURE is the word 'While' followed by the Gherkin Given condition
$SYSTEM_RESPONSE is a combination of the Actors and the Gherkin Then condition.
$ACTOR, $MODAL_VERB and a $SYSTEM_RESPONSE is mandatory in all Rimay requirements.
Replace the variables with the right text according the input acceptance criteria.
In the following order:
Rimay CNL definition: $SYSTEM_RESPONSE $MODAL_VERB $WHILE_STRUCTURE $ACTOR $WHEN_STRUCTURE
With the usage of the following extra information:
Actors: actor SystemA, actor SystemB, actor SystemC, actor UserX, actor UserA, actor UserC
### Input Data
Use the following Gherkin acceptance criteria input:
```
Given: channel "United States" uses "maverick/meerkat" theme
When: UserA visit this channel's homepage
Then: UserA should see a homepage from that theme
```
### Question:
Can you translate the input above into the specified language Rimay CNL?
### Output indicator
Output only the translated Rimay CNL text and nothing else!
## Rimay Output
```
UserA should see a homepage from that theme should While UserA visit this channel's homepage UserA
```
### Researcher score
Researcher_score = -10
## DSL-Rimay Verification
```
ERROR:no viable alternative at input 'should' (line : 3 column : 7)
ERROR:missing '.' at 'see' (line : 3 column : 14)
ERROR:no viable alternative at input 'a' (line : 3 column : 18)
ERROR:missing '.' at 'homepage' (line : 3 column : 20)
ERROR:no viable alternative at input 'from' (line : 3 column : 29)
ERROR:no viable alternative at input 'theme' (line : 3 column : 39)
ERROR:no viable alternative at input 'should' (line : 3 column : 45)
ERROR:missing '.' at 'While' (line : 3 column : 52)
ERROR:no viable alternative at input 'this' (line : 3 column : 70)
ERROR:no viable alternative at input 'channel' (line : 3 column : 75)
ERROR:mismatched character '<EOF>' expecting ''' (line : 3 column : 82)
ERROR:Couldn't resolve reference to Actor 'see'. (line : 3 column : 14)
ERROR:Couldn't resolve reference to Actor 'homepage'. (line : 3 column : 20)
ERROR:Couldn't resolve reference to Actor 'theme'. (line : 3 column : 39)
ERROR:Couldn't resolve reference to ActorOrClassOrProperty 'visit'. (line : 3 column : 64)
ERROR:Couldn't resolve reference to Quantifier 'UserA'. (line : 3 column : 58)
ERROR:Couldn't resolve reference to Actor 'this'. (line : 3 column : 70)
```
### DSL-Rimay Score
DSL_Score: 67
Progressbar: ![67%](https://progress-bar.dev/67)
## Rimay Paska Verification
```
File generated_rimay.csv.csv
Smells found in this document:
Non-atomic requirement: 0
Incomplete requirement: 0
Incorrect order requirement: 1
Coordination ambiguity: 0
Not requirement: ,0
Incomplete condition: 0
Incomplete system response: 0
Passive voice: 0
Not precise verb: 0
Number Req with smells: 1
```
### Paska Score
PASKA_Score: 95
Progressbar: ![95%](https://progress-bar.dev/95)
### Final Score
Final_score = 71.0
Progressbar: ![71.0%](https://progress-bar.dev/71.0)
output_dataset/Few-shot-learning_0.2temp/Gherkinscenario 15: Displaying themed shop homepage__04_05_2024__14:06:38__96.0_score.md
+131
View File
@@ -0,0 +1,131 @@
# LLM Experiment Information
* Prefix: Few-shot-learning
* Datetime: 04_05_2024__14:06:38
* UniqueID: 39c2baba-cf96-4031-9378-2e74a1ba9f5d
* Gherkinscenario name: Gherkinscenario 15: Displaying themed shop homepage
* LLM-temperature: 0.2
## Gherkin Input
```
Given: channel "United States" uses "maverick/meerkat" theme
When: UserA visit this channel's homepage
Then: UserA should see a homepage from that theme
```
## Input prompt, technique: Few-Shot-Learning
### Context
We are translating Gherkin acceptance criteria into Rimay CNL.
For this translation use the following information:
$ACTOR is for example a thing or a person.
$MODAL_VERB is for the following word: must or shall.
$TRIGGER is for example a trigger action, something what a system does.
$WHILE_STRUCTURE is the word 'While' followed by the Gherkin Given condition
$WHEN_STRUCTURE is the word 'When' followed by the Gherkin When condition
$SYSTEM_RESPONSE is a combination of the Actors and the Gherkin Then condition.
$ACTOR, $MODAL_VERB and a $SYSTEM_RESPONSE is mandatory in all Rimay requirements.
Replace the variables with the right text according the input acceptance criteria.
In the following order:
Rimay CNL definition: $WHILE_STRUCTURE $WHEN_STRUCTURE $ACTOR $MODAL_VERB $SYSTEM_RESPONSE
The following example is a valid Rimay CNL:
When SystemB receives an "email alert" from SystemA, SystemB must send an Instruction to SystemC.
With the usage of the following extra information:
Actors: actor SystemA, actor SystemB, actor SystemC, actor UserX, actor UserA, actor UserC
### Input Data
Use the following Gherkin acceptance criteria input:
```
Given: channel "United States" uses "maverick/meerkat" theme
When: UserA visit this channel's homepage
Then: UserA should see a homepage from that theme
```
### Question:
Can you translate the input above into the specified language Rimay CNL?
### Output indicator
Output only the translated Rimay CNL text and nothing else!
## Rimay Output
```
While channel "United States" uses "maverick/meerkat" theme When UserA visit this channel's homepage UserA must UserA should see a homepage from that theme
```
### Researcher score
Researcher_score = 10
## DSL-Rimay Verification
```
ERROR:no viable alternative at input 'uses' (line : 3 column : 7)
ERROR:no viable alternative at input '"United States"' (line : 3 column : 15)
ERROR:missing '.' at 'theme' (line : 3 column : 55)
ERROR:no viable alternative at input 'When' (line : 3 column : 63)
ERROR:missing '.' at 'UserA' (line : 3 column : 68)
ERROR:no viable alternative at input 'visit' (line : 3 column : 74)
ERROR:missing '.' at 'this' (line : 3 column : 80)
ERROR:no viable alternative at input 'channel' (line : 3 column : 85)
ERROR:mismatched character '<EOF>' expecting ''' (line : 3 column : 92)
ERROR:Couldn't resolve reference to Actor 'channel'. (line : 3 column : 7)
ERROR:Couldn't resolve reference to Actor 'theme'. (line : 3 column : 55)
ERROR:Couldn't resolve reference to Actor 'this'. (line : 3 column : 80)
```
### DSL-Rimay Score
DSL_Score: 72
Progressbar: ![72%](https://progress-bar.dev/72)
## Rimay Paska Verification
```
File generated_rimay.csv.csv
Smells found in this document:
Non-atomic requirement: 0
Incomplete requirement: 0
Incorrect order requirement: 0
Coordination ambiguity: 0
Not requirement: ,0
Incomplete condition: 0
Incomplete system response: 0
Passive voice: 0
Not precise verb: 0
Number Req with smells: 0
```
### Paska Score
PASKA_Score: 100
Progressbar: ![100%](https://progress-bar.dev/100)
### Final Score
Final_score = 96.0
Progressbar: ![96.0%](https://progress-bar.dev/96.0)
output_dataset/Few-shot-learning_0.2temp/Gherkinscenario 16: Choosing shipping address from address book_Incorrect___04_05_2024__14:09:52__74.5_score.md
+135
View File
@@ -0,0 +1,135 @@
# LLM Experiment Information
* Prefix: Few-shot-learning
* Datetime: 04_05_2024__14:09:52
* UniqueID: 61e56943-aa01-4012-8a5c-05334ffac5e1
* Gherkinscenario name: Gherkinscenario 16: Choosing shipping address from address book_Incorrect_
* LLM-temperature: 0.2
## Gherkin Input
```
Given: UserA has product "PHP T-Shirt" in the cart
When: UserA chooses "Seaside Fwy" street for shipping address
Then: address "Lucifer Morningstar, Seaside Fwy, 90802, Los Angeles, United States, Arkansas" should be filled as shipping address
```
## Input prompt, technique: Few-Shot-Learning
### Context
We are translating Gherkin acceptance criteria into Rimay CNL.
For this translation use the following information:
$ACTOR is for example a thing or a person.
$MODAL_VERB is for the following word: should or can
$TRIGGER is for example a trigger action, nothing what a system does.
$WHILE_STRUCTUREis the word 'When' followed by the Gherkin When condition
$WHEN_STRUCTURE is the word 'While' followed by the Gherkin Given condition
$SYSTEM_RESPONSE is a combination of the Actors and the Gherkin Then condition.
$ACTOR, $MODAL_VERB and a $SYSTEM_RESPONSE is mandatory in all Rimay requirements.
Replace the variables with the right text according the input acceptance criteria.
In the following order:
Rimay CNL definition: $SYSTEM_RESPONSE $MODAL_VERB $WHILE_STRUCTURE $ACTOR $WHEN_STRUCTURE
With the usage of the following extra information:
Actors: actor SystemA, actor SystemB, actor SystemC, actor UserX, actor UserA, actor UserC
### Input Data
Use the following Gherkin acceptance criteria input:
```
Given: UserA has product "PHP T-Shirt" in the cart
When: UserA chooses "Seaside Fwy" street for shipping address
Then: address "Lucifer Morningstar, Seaside Fwy, 90802, Los Angeles, United States, Arkansas" should be filled as shipping address
```
### Question:
Can you translate the input above into the specified language Rimay CNL?
### Output indicator
Output only the translated Rimay CNL text and nothing else!
## Rimay Output
```
address "Lucifer Morningstar, Seaside Fwy, 90802, Los Angeles, United States, Arkansas" should UserA While UserA has product "PHP T-Shirt" in the cart SystemA When UserA chooses "Seaside Fwy" street for shipping address
```
### Researcher score
Researcher_score = -10
## DSL-Rimay Verification
```
ERROR:no viable alternative at input '"Lucifer Morningstar, Seaside Fwy, 90802, Los Angeles, United States, Arkansas"' (line : 3 column : 9)
ERROR:no viable alternative at input 'should' (line : 3 column : 89)
ERROR:no viable alternative at input 'UserA' (line : 3 column : 96)
ERROR:missing '.' at 'While' (line : 3 column : 107)
ERROR:mismatched input 'in' expecting ',' (line : 3 column : 145)
ERROR:missing '.' at 'the' (line : 3 column : 148)
ERROR:no viable alternative at input 'SystemA' (line : 3 column : 162)
ERROR:missing '.' at 'When' (line : 3 column : 175)
ERROR:no viable alternative at input 'street' (line : 3 column : 208)
ERROR:no viable alternative at input 'for' (line : 3 column : 215)
ERROR:no viable alternative at input 'shipping' (line : 3 column : 219)
ERROR:no viable alternative at input 'address' (line : 3 column : 228)
ERROR:Couldn't resolve reference to Actor 'address'. (line : 3 column : 1)
ERROR:Couldn't resolve reference to Actor 'should'. (line : 3 column : 89)
ERROR:Couldn't resolve reference to Quantifier 'product'. (line : 3 column : 123)
ERROR:Couldn't resolve reference to Actor 'cart'. (line : 3 column : 152)
ERROR:Couldn't resolve reference to Actor 'chooses'. (line : 3 column : 186)
ERROR:Couldn't resolve reference to Quantifier 'UserA'. (line : 3 column : 180)
ERROR:Couldn't resolve reference to Actor 'street'. (line : 3 column : 208)
ERROR:Couldn't resolve reference to Actor 'shipping'. (line : 3 column : 219)
```
### DSL-Rimay Score
DSL_Score: 74
Progressbar: ![74%](https://progress-bar.dev/74)
## Rimay Paska Verification
```
File generated_rimay.csv.csv
Smells found in this document:
Non-atomic requirement: 0
Incomplete requirement: 0
Incorrect order requirement: 1
Coordination ambiguity: 0
Not requirement: ,0
Incomplete condition: 0
Incomplete system response: 0
Passive voice: 0
Not precise verb: 0
Number Req with smells: 1
```
### Paska Score
PASKA_Score: 95
Progressbar: ![95%](https://progress-bar.dev/95)
### Final Score
Final_score = 74.5
Progressbar: ![74.5%](https://progress-bar.dev/74.5)
output_dataset/Few-shot-learning_0.2temp/Gherkinscenario 16: Choosing shipping address from address book__04_05_2024__14:08:42__89.0_score.md
+145
View File
@@ -0,0 +1,145 @@
# LLM Experiment Information
* Prefix: Few-shot-learning
* Datetime: 04_05_2024__14:08:42
* UniqueID: 7ad18fc5-a4de-4dfb-81b6-23dc3898baaa
* Gherkinscenario name: Gherkinscenario 16: Choosing shipping address from address book
* LLM-temperature: 0.2
## Gherkin Input
```
Given: UserA has product "PHP T-Shirt" in the cart
When: UserA chooses "Seaside Fwy" street for shipping address
Then: address "Lucifer Morningstar, Seaside Fwy, 90802, Los Angeles, United States, Arkansas" should be filled as shipping address
```
## Input prompt, technique: Few-Shot-Learning
### Context
We are translating Gherkin acceptance criteria into Rimay CNL.
For this translation use the following information:
$ACTOR is for example a thing or a person.
$MODAL_VERB is for the following word: must or shall.
$TRIGGER is for example a trigger action, something what a system does.
$WHILE_STRUCTURE is the word 'While' followed by the Gherkin Given condition
$WHEN_STRUCTURE is the word 'When' followed by the Gherkin When condition
$SYSTEM_RESPONSE is a combination of the Actors and the Gherkin Then condition.
$ACTOR, $MODAL_VERB and a $SYSTEM_RESPONSE is mandatory in all Rimay requirements.
Replace the variables with the right text according the input acceptance criteria.
In the following order:
Rimay CNL definition: $WHILE_STRUCTURE $WHEN_STRUCTURE $ACTOR $MODAL_VERB $SYSTEM_RESPONSE
The following example is a valid Rimay CNL:
When SystemB receives an "email alert" from SystemA, SystemB must send an Instruction to SystemC.
With the usage of the following extra information:
Actors: actor SystemA, actor SystemB, actor SystemC, actor UserX, actor UserA, actor UserC
### Input Data
Use the following Gherkin acceptance criteria input:
```
Given: UserA has product "PHP T-Shirt" in the cart
When: UserA chooses "Seaside Fwy" street for shipping address
Then: address "Lucifer Morningstar, Seaside Fwy, 90802, Los Angeles, United States, Arkansas" should be filled as shipping address
```
### Question:
Can you translate the input above into the specified language Rimay CNL?
### Output indicator
Output only the translated Rimay CNL text and nothing else!
## Rimay Output
```
While UserA has product "PHP T-Shirt" in the cart When UserA chooses "Seaside Fwy" street for shipping address UserA must address "Lucifer Morningstar, Seaside Fwy, 90802, Los Angeles, United States, Arkansas" should be filled as shipping address
```
### Researcher score
Researcher_score = 10
## DSL-Rimay Verification
```
ERROR:mismatched input 'in' expecting ',' (line : 3 column : 39)
ERROR:missing '.' at 'the' (line : 3 column : 42)
ERROR:no viable alternative at input 'When' (line : 3 column : 53)
ERROR:missing '.' at 'UserA' (line : 3 column : 58)
ERROR:no viable alternative at input 'chooses' (line : 3 column : 64)
ERROR:missing '.' at 'street' (line : 3 column : 86)
ERROR:no viable alternative at input 'for' (line : 3 column : 93)
ERROR:no viable alternative at input 'shipping' (line : 3 column : 97)
ERROR:no viable alternative at input 'address' (line : 3 column : 106)
ERROR:missing '.' at 'UserA' (line : 3 column : 116)
ERROR:no viable alternative at input 'address' (line : 3 column : 131)
ERROR:no viable alternative at input '"Lucifer Morningstar, Seaside Fwy, 90802, Los Angeles, United States, Arkansas"' (line : 3 column : 139)
ERROR:no viable alternative at input 'should' (line : 3 column : 219)
ERROR:no viable alternative at input 'be' (line : 3 column : 226)
ERROR:missing '.' at 'filled' (line : 3 column : 229)
ERROR:no viable alternative at input 'as' (line : 3 column : 236)
ERROR:no viable alternative at input 'shipping' (line : 3 column : 239)
ERROR:no viable alternative at input 'address' (line : 3 column : 248)
ERROR:Couldn't resolve reference to Quantifier 'product'. (line : 3 column : 17)
ERROR:Couldn't resolve reference to Actor 'cart'. (line : 3 column : 46)
ERROR:Couldn't resolve reference to Actor 'street'. (line : 3 column : 86)
ERROR:Couldn't resolve reference to Actor 'shipping'. (line : 3 column : 97)
ERROR:Couldn't resolve reference to Actor 'address'. (line : 3 column : 131)
ERROR:Couldn't resolve reference to Actor 'should'. (line : 3 column : 219)
ERROR:Couldn't resolve reference to Actor 'filled'. (line : 3 column : 229)
ERROR:Couldn't resolve reference to Actor 'shipping'. (line : 3 column : 239)
```
### DSL-Rimay Score
DSL_Score: 68
Progressbar: ![68%](https://progress-bar.dev/68)
## Rimay Paska Verification
```
File generated_rimay.csv.csv
Smells found in this document:
Non-atomic requirement: 1
Incomplete requirement: 0
Incorrect order requirement: 0
Coordination ambiguity: 0
Not requirement: ,0
Incomplete condition: 0
Incomplete system response: 0
Passive voice: 1
Not precise verb: 0
Number Req with smells: 1
```
### Paska Score
PASKA_Score: 90
Progressbar: ![90%](https://progress-bar.dev/90)
### Final Score
Final_score = 89.0
Progressbar: ![89.0%](https://progress-bar.dev/89.0)
output_dataset/Few-shot-learning_0.2temp/Gherkinscenario 17: Seeing the unpaid order in the list_Incorrect___04_05_2024__14:11:51__68.5_score.md
+127
View File
@@ -0,0 +1,127 @@
# LLM Experiment Information
* Prefix: Few-shot-learning
* Datetime: 04_05_2024__14:11:51
* UniqueID: daaa671d-2bf4-4af8-8b84-ab19adb71294
* Gherkinscenario name: Gherkinscenario 17: Seeing the unpaid order in the list_Incorrect_
* LLM-temperature: 0.2
## Gherkin Input
```
Given: UserA logged in as "john@example.com"
When: UserA views order history
Then: UserA should sees one order with total of $29.99
```
## Input prompt, technique: Few-Shot-Learning
### Context
We are translating Gherkin acceptance criteria into Rimay CNL.
For this translation use the following information:
$ACTOR is for example a thing or a person.
$MODAL_VERB is for the following word: should or can
$TRIGGER is for example a trigger action, nothing what a system does.
$WHILE_STRUCTUREis the word 'When' followed by the Gherkin When condition
$WHEN_STRUCTURE is the word 'While' followed by the Gherkin Given condition
$SYSTEM_RESPONSE is a combination of the Actors and the Gherkin Then condition.
$ACTOR, $MODAL_VERB and a $SYSTEM_RESPONSE is mandatory in all Rimay requirements.
Replace the variables with the right text according the input acceptance criteria.
In the following order:
Rimay CNL definition: $SYSTEM_RESPONSE $MODAL_VERB $WHILE_STRUCTURE $ACTOR $WHEN_STRUCTURE
With the usage of the following extra information:
Actors: actor SystemA, actor SystemB, actor SystemC, actor UserX, actor UserA, actor UserC
### Input Data
Use the following Gherkin acceptance criteria input:
```
Given: UserA logged in as "john@example.com"
When: UserA views order history
Then: UserA should sees one order with total of $29.99
```
### Question:
Can you translate the input above into the specified language Rimay CNL?
### Output indicator
Output only the translated Rimay CNL text and nothing else!
## Rimay Output
```
UserA should sees one order with total of $29.99 should UserA While UserA views order history
```
### Researcher score
Researcher_score = -10
## DSL-Rimay Verification
```
ERROR:no viable alternative at input 'should' (line : 3 column : 7)
ERROR:missing '.' at 'sees' (line : 3 column : 14)
ERROR:no viable alternative at input 'one' (line : 3 column : 19)
ERROR:missing '.' at 'order' (line : 3 column : 23)
ERROR:no viable alternative at input 'with' (line : 3 column : 29)
ERROR:no viable alternative at input 'total' (line : 3 column : 34)
ERROR:no viable alternative at input 'of' (line : 3 column : 40)
ERROR:no viable alternative at input '.' (line : 3 column : 46)
ERROR:missing EOF at '99' (line : 3 column : 47)
ERROR:Couldn't resolve reference to Actor 'sees'. (line : 3 column : 14)
ERROR:Couldn't resolve reference to Actor 'order'. (line : 3 column : 23)
ERROR:Couldn't resolve reference to Actor 'total'. (line : 3 column : 34)
```
### DSL-Rimay Score
DSL_Score: 72
Progressbar: ![72%](https://progress-bar.dev/72)
## Rimay Paska Verification
```
File generated_rimay.csv.csv
Smells found in this document:
Non-atomic requirement: 0
Incomplete requirement: 0
Incorrect order requirement: 1
Coordination ambiguity: 0
Not requirement: ,0
Incomplete condition: 1
Incomplete system response: 0
Passive voice: 0
Not precise verb: 0
Number Req with smells: 1
```
### Paska Score
PASKA_Score: 85
Progressbar: ![85%](https://progress-bar.dev/85)
### Final Score
Final_score = 68.5
Progressbar: ![68.5%](https://progress-bar.dev/68.5)
output_dataset/Few-shot-learning_0.2temp/Gherkinscenario 17: Seeing the unpaid order in the list__04_05_2024__14:10:45__76.5_score.md
+140
View File
@@ -0,0 +1,140 @@
# LLM Experiment Information
* Prefix: Few-shot-learning
* Datetime: 04_05_2024__14:10:45
* UniqueID: 2cbc372a-6ea3-426b-a6cb-9a984b336f12
* Gherkinscenario name: Gherkinscenario 17: Seeing the unpaid order in the list
* LLM-temperature: 0.2
## Gherkin Input
```
Given: UserA logged in as "john@example.com"
When: UserA views order history
Then: UserA should sees one order with total of $29.99
```
## Input prompt, technique: Few-Shot-Learning
### Context
We are translating Gherkin acceptance criteria into Rimay CNL.
For this translation use the following information:
$ACTOR is for example a thing or a person.
$MODAL_VERB is for the following word: must or shall.
$TRIGGER is for example a trigger action, something what a system does.
$WHILE_STRUCTURE is the word 'While' followed by the Gherkin Given condition
$WHEN_STRUCTURE is the word 'When' followed by the Gherkin When condition
$SYSTEM_RESPONSE is a combination of the Actors and the Gherkin Then condition.
$ACTOR, $MODAL_VERB and a $SYSTEM_RESPONSE is mandatory in all Rimay requirements.
Replace the variables with the right text according the input acceptance criteria.
In the following order:
Rimay CNL definition: $WHILE_STRUCTURE $WHEN_STRUCTURE $ACTOR $MODAL_VERB $SYSTEM_RESPONSE
The following example is a valid Rimay CNL:
When SystemB receives an "email alert" from SystemA, SystemB must send an Instruction to SystemC.
With the usage of the following extra information:
Actors: actor SystemA, actor SystemB, actor SystemC, actor UserX, actor UserA, actor UserC
### Input Data
Use the following Gherkin acceptance criteria input:
```
Given: UserA logged in as "john@example.com"
When: UserA views order history
Then: UserA should sees one order with total of $29.99
```
### Question:
Can you translate the input above into the specified language Rimay CNL?
### Output indicator
Output only the translated Rimay CNL text and nothing else!
## Rimay Output
```
While UserA is logged in as "john@example.com" When UserA views order history UserA must UserA should sees one order with total of $29.99
```
### Researcher score
Researcher_score = 10
## DSL-Rimay Verification
```
ERROR:no viable alternative at input 'in' (line : 3 column : 23)
ERROR:no viable alternative at input 'order' (line : 3 column : 67)
ERROR:no viable alternative at input 'history' (line : 3 column : 73)
ERROR:missing '.' at 'UserA' (line : 3 column : 83)
ERROR:no viable alternative at input 'UserA' (line : 3 column : 98)
ERROR:no viable alternative at input 'should' (line : 3 column : 104)
ERROR:missing '.' at 'sees' (line : 3 column : 111)
ERROR:no viable alternative at input 'one' (line : 3 column : 116)
ERROR:missing '.' at 'order' (line : 3 column : 120)
ERROR:no viable alternative at input 'with' (line : 3 column : 126)
ERROR:no viable alternative at input 'total' (line : 3 column : 131)
ERROR:no viable alternative at input 'of' (line : 3 column : 137)
ERROR:no viable alternative at input '.' (line : 3 column : 143)
ERROR:extraneous input '99' expecting EOF (line : 3 column : 144)
ERROR:Couldn't resolve reference to Quantifier 'logged'. (line : 3 column : 16)
ERROR:Couldn't resolve reference to Actor 'views'. (line : 3 column : 61)
ERROR:Couldn't resolve reference to Quantifier 'UserA'. (line : 3 column : 55)
ERROR:Couldn't resolve reference to Actor 'order'. (line : 3 column : 67)
ERROR:Couldn't resolve reference to Actor 'sees'. (line : 3 column : 111)
ERROR:Couldn't resolve reference to Actor 'order'. (line : 3 column : 120)
ERROR:Couldn't resolve reference to Actor 'total'. (line : 3 column : 131)
```
### DSL-Rimay Score
DSL_Score: 63
Progressbar: ![63%](https://progress-bar.dev/63)
## Rimay Paska Verification
```
File generated_rimay.csv.csv
Smells found in this document:
Non-atomic requirement: 0
Incomplete requirement: 1
Incorrect order requirement: 0
Coordination ambiguity: 0
Not requirement: ,0
Incomplete condition: 0
Incomplete system response: 0
Passive voice: 1
Not precise verb: 0
Number Req with smells: 1
```
### Paska Score
PASKA_Score: 70
Progressbar: ![70%](https://progress-bar.dev/70)
### Final Score
Final_score = 76.5
Progressbar: ![76.5%](https://progress-bar.dev/76.5)
output_dataset/Few-shot-learning_0.2temp/Gherkinscenario 18: Being unable to change currencies_Incorrect___04_05_2024__14:13:43__77.0_score.md
+135
View File
@@ -0,0 +1,135 @@
# LLM Experiment Information
* Prefix: Few-shot-learning
* Datetime: 04_05_2024__14:13:43
* UniqueID: 52b094b4-cce7-45f6-a002-fa93b7cb2dec
* Gherkinscenario name: Gherkinscenario 18: Being unable to change currencies_Incorrect_
* LLM-temperature: 0.2
## Gherkin Input
```
Given: the exchange rate of "US Dollar" to "British Pound" is 1.30
When: UserA wants to edit this exchange rate
Then: UserA should see that the source currency is disabled
```
## Input prompt, technique: Few-Shot-Learning
### Context
We are translating Gherkin acceptance criteria into Rimay CNL.
For this translation use the following information:
$ACTOR is for example a thing or a person.
$MODAL_VERB is for the following word: should or can
$TRIGGER is for example a trigger action, nothing what a system does.
$WHILE_STRUCTUREis the word 'When' followed by the Gherkin When condition
$WHEN_STRUCTURE is the word 'While' followed by the Gherkin Given condition
$SYSTEM_RESPONSE is a combination of the Actors and the Gherkin Then condition.
$ACTOR, $MODAL_VERB and a $SYSTEM_RESPONSE is mandatory in all Rimay requirements.
Replace the variables with the right text according the input acceptance criteria.
In the following order:
Rimay CNL definition: $SYSTEM_RESPONSE $MODAL_VERB $WHILE_STRUCTURE $ACTOR $WHEN_STRUCTURE
With the usage of the following extra information:
Actors: actor SystemA, actor SystemB, actor SystemC, actor UserX, actor UserA, actor UserC
### Input Data
Use the following Gherkin acceptance criteria input:
```
Given: the exchange rate of "US Dollar" to "British Pound" is 1.30
When: UserA wants to edit this exchange rate
Then: UserA should see that the source currency is disabled
```
### Question:
Can you translate the input above into the specified language Rimay CNL?
### Output indicator
Output only the translated Rimay CNL text and nothing else!
## Rimay Output
```
UserA should see that the source currency is disabled should While UserA wants to edit this exchange rate
```
### Researcher score
Researcher_score = -5
## DSL-Rimay Verification
```
ERROR:no viable alternative at input 'should' (line : 3 column : 7)
ERROR:missing '.' at 'see' (line : 3 column : 14)
ERROR:no viable alternative at input 'that' (line : 3 column : 18)
ERROR:no viable alternative at input 'the' (line : 3 column : 23)
ERROR:no viable alternative at input 'currency' (line : 3 column : 34)
ERROR:missing '.' at 'disabled' (line : 3 column : 46)
ERROR:no viable alternative at input 'should' (line : 3 column : 55)
ERROR:missing '.' at 'While' (line : 3 column : 62)
ERROR:no viable alternative at input 'to' (line : 3 column : 80)
ERROR:missing ',' at 'edit' (line : 3 column : 83)
ERROR:no viable alternative at input 'this' (line : 3 column : 88)
ERROR:missing '.' at 'exchange' (line : 3 column : 93)
ERROR:no viable alternative at input 'rate' (line : 3 column : 102)
ERROR:Couldn't resolve reference to Actor 'see'. (line : 3 column : 14)
ERROR:Couldn't resolve reference to Actor 'source'. (line : 3 column : 27)
ERROR:Couldn't resolve reference to Actor 'disabled'. (line : 3 column : 46)
ERROR:Couldn't resolve reference to ActorOrClassOrProperty 'wants'. (line : 3 column : 74)
ERROR:Couldn't resolve reference to Quantifier 'UserA'. (line : 3 column : 68)
ERROR:Couldn't resolve reference to Actor 'edit'. (line : 3 column : 83)
ERROR:Couldn't resolve reference to Actor 'exchange'. (line : 3 column : 93)
```
### DSL-Rimay Score
DSL_Score: 74
Progressbar: ![74%](https://progress-bar.dev/74)
## Rimay Paska Verification
```
File generated_rimay.csv.csv
Smells found in this document:
Non-atomic requirement: 0
Incomplete requirement: 0
Incorrect order requirement: 1
Coordination ambiguity: 0
Not requirement: ,0
Incomplete condition: 0
Incomplete system response: 0
Passive voice: 0
Not precise verb: 1
Number Req with smells: 1
```
### Paska Score
PASKA_Score: 90
Progressbar: ![90%](https://progress-bar.dev/90)
### Final Score
Final_score = 77.0
Progressbar: ![77.0%](https://progress-bar.dev/77.0)
output_dataset/Few-shot-learning_0.2temp/Gherkinscenario 18: Being unable to change currencies__04_05_2024__14:12:44__100_score.md
+136
View File
@@ -0,0 +1,136 @@
# LLM Experiment Information
* Prefix: Few-shot-learning
* Datetime: 04_05_2024__14:12:44
* UniqueID: d3c4b60e-3b50-49d9-9274-13604d78ff04
* Gherkinscenario name: Gherkinscenario 18: Being unable to change currencies
* LLM-temperature: 0.2
## Gherkin Input
```
Given: the exchange rate of "US Dollar" to "British Pound" is 1.30
When: UserA wants to edit this exchange rate
Then: UserA should see that the source currency is disabled
```
## Input prompt, technique: Few-Shot-Learning
### Context
We are translating Gherkin acceptance criteria into Rimay CNL.
For this translation use the following information:
$ACTOR is for example a thing or a person.
$MODAL_VERB is for the following word: must or shall.
$TRIGGER is for example a trigger action, something what a system does.
$WHILE_STRUCTURE is the word 'While' followed by the Gherkin Given condition
$WHEN_STRUCTURE is the word 'When' followed by the Gherkin When condition
$SYSTEM_RESPONSE is a combination of the Actors and the Gherkin Then condition.
$ACTOR, $MODAL_VERB and a $SYSTEM_RESPONSE is mandatory in all Rimay requirements.
Replace the variables with the right text according the input acceptance criteria.
In the following order:
Rimay CNL definition: $WHILE_STRUCTURE $WHEN_STRUCTURE $ACTOR $MODAL_VERB $SYSTEM_RESPONSE
The following example is a valid Rimay CNL:
When SystemB receives an "email alert" from SystemA, SystemB must send an Instruction to SystemC.
With the usage of the following extra information:
Actors: actor SystemA, actor SystemB, actor SystemC, actor UserX, actor UserA, actor UserC
### Input Data
Use the following Gherkin acceptance criteria input:
```
Given: the exchange rate of "US Dollar" to "British Pound" is 1.30
When: UserA wants to edit this exchange rate
Then: UserA should see that the source currency is disabled
```
### Question:
Can you translate the input above into the specified language Rimay CNL?
### Output indicator
Output only the translated Rimay CNL text and nothing else!
## Rimay Output
```
When UserA wants to edit the exchange rate of "US Dollar" to "British Pound", UserA must see that the source currency is disabled.
```
### Researcher score
Researcher_score = 15
## DSL-Rimay Verification
```
ERROR:no viable alternative at input 'to' (line : 3 column : 18)
ERROR:missing ',' at 'edit' (line : 3 column : 21)
ERROR:no viable alternative at input 'the' (line : 3 column : 26)
ERROR:missing '.' at 'exchange' (line : 3 column : 30)
ERROR:no viable alternative at input 'rate' (line : 3 column : 39)
ERROR:missing '.' at 'UserA' (line : 3 column : 79)
ERROR:no viable alternative at input 'see' (line : 3 column : 90)
ERROR:no viable alternative at input 'that' (line : 3 column : 94)
ERROR:no viable alternative at input 'the' (line : 3 column : 99)
ERROR:no viable alternative at input 'currency' (line : 3 column : 110)
ERROR:extraneous input 'disabled' expecting '.' (line : 3 column : 122)
ERROR:Couldn't resolve reference to Actor 'wants'. (line : 3 column : 12)
ERROR:Couldn't resolve reference to Quantifier 'UserA'. (line : 3 column : 6)
ERROR:Couldn't resolve reference to Actor 'edit'. (line : 3 column : 21)
ERROR:Couldn't resolve reference to Actor 'exchange'. (line : 3 column : 30)
ERROR:Couldn't resolve reference to Actor 'see'. (line : 3 column : 90)
ERROR:Couldn't resolve reference to Actor 'source'. (line : 3 column : 103)
```
### DSL-Rimay Score
DSL_Score: 77
Progressbar: ![77%](https://progress-bar.dev/77)
## Rimay Paska Verification
```
File generated_rimay.csv.csv
Smells found in this document:
Non-atomic requirement: 0
Incomplete requirement: 0
Incorrect order requirement: 0
Coordination ambiguity: 0
Not requirement: ,0
Incomplete condition: 0
Incomplete system response: 0
Passive voice: 0
Not precise verb: 0
Number Req with smells: 0
```
### Paska Score
PASKA_Score: 100
Progressbar: ![100%](https://progress-bar.dev/100)
### Final Score
Final_score = 100
Progressbar: ![100%](https://progress-bar.dev/100)
output_dataset/Few-shot-learning_0.2temp/Gherkinscenario 19: Inability to impersonate a customer with no account_Incorrect___04_05_2024__14:15:34__73.5_score.md
+137
View File
@@ -0,0 +1,137 @@
# LLM Experiment Information
* Prefix: Few-shot-learning
* Datetime: 04_05_2024__14:15:34
* UniqueID: aa33d710-4abe-4b9d-b076-d8decfab7145
* Gherkinscenario name: Gherkinscenario 19: Inability to impersonate a customer with no account_Incorrect_
* LLM-temperature: 0.2
## Gherkin Input
```
Given: SystemA has customer "harold@thrasher.ie" with first name "Harold"
When: UserA views their details
Then: UserA should be unable to impersonate them
```
## Input prompt, technique: Few-Shot-Learning
### Context
We are translating Gherkin acceptance criteria into Rimay CNL.
For this translation use the following information:
$ACTOR is for example a thing or a person.
$MODAL_VERB is for the following word: should or can
$TRIGGER is for example a trigger action, nothing what a system does.
$WHILE_STRUCTUREis the word 'When' followed by the Gherkin When condition
$WHEN_STRUCTURE is the word 'While' followed by the Gherkin Given condition
$SYSTEM_RESPONSE is a combination of the Actors and the Gherkin Then condition.
$ACTOR, $MODAL_VERB and a $SYSTEM_RESPONSE is mandatory in all Rimay requirements.
Replace the variables with the right text according the input acceptance criteria.
In the following order:
Rimay CNL definition: $SYSTEM_RESPONSE $MODAL_VERB $WHILE_STRUCTURE $ACTOR $WHEN_STRUCTURE
With the usage of the following extra information:
Actors: actor SystemA, actor SystemB, actor SystemC, actor UserX, actor UserA, actor UserC
### Input Data
Use the following Gherkin acceptance criteria input:
```
Given: SystemA has customer "harold@thrasher.ie" with first name "Harold"
When: UserA views their details
Then: UserA should be unable to impersonate them
```
### Question:
Can you translate the input above into the specified language Rimay CNL?
### Output indicator
Output only the translated Rimay CNL text and nothing else!
## Rimay Output
```
UserA should be unable to impersonate them should SystemA has customer "harold@thrasher.ie" with first name "Harold" While UserA views their details
```
### Researcher score
Researcher_score = -10
## DSL-Rimay Verification
```
ERROR:no viable alternative at input 'should' (line : 3 column : 7)
ERROR:missing '.' at 'be' (line : 3 column : 14)
ERROR:no viable alternative at input 'unable' (line : 3 column : 17)
ERROR:missing '.' at 'impersonate' (line : 3 column : 27)
ERROR:no viable alternative at input 'them' (line : 3 column : 39)
ERROR:missing '.' at 'should' (line : 3 column : 44)
ERROR:no viable alternative at input 'SystemA' (line : 3 column : 51)
ERROR:missing '.' at 'customer' (line : 3 column : 63)
ERROR:no viable alternative at input '"harold@thrasher.ie"' (line : 3 column : 72)
ERROR:no viable alternative at input 'first' (line : 3 column : 98)
ERROR:no viable alternative at input 'name' (line : 3 column : 104)
ERROR:missing '.' at 'While' (line : 3 column : 118)
ERROR:no viable alternative at input 'their' (line : 3 column : 136)
ERROR:no viable alternative at input 'details' (line : 3 column : 142)
ERROR:Couldn't resolve reference to Actor 'be'. (line : 3 column : 14)
ERROR:Couldn't resolve reference to Actor 'impersonate'. (line : 3 column : 27)
ERROR:Couldn't resolve reference to Actor 'should'. (line : 3 column : 44)
ERROR:Couldn't resolve reference to Actor 'customer'. (line : 3 column : 63)
ERROR:Couldn't resolve reference to Actor 'first'. (line : 3 column : 98)
ERROR:Couldn't resolve reference to ActorOrClassOrProperty 'views'. (line : 3 column : 130)
ERROR:Couldn't resolve reference to Quantifier 'UserA'. (line : 3 column : 124)
ERROR:Couldn't resolve reference to Actor 'their'. (line : 3 column : 136)
```
### DSL-Rimay Score
DSL_Score: 72
Progressbar: ![72%](https://progress-bar.dev/72)
## Rimay Paska Verification
```
File generated_rimay.csv.csv
Smells found in this document:
Non-atomic requirement: 0
Incomplete requirement: 0
Incorrect order requirement: 1
Coordination ambiguity: 0
Not requirement: ,0
Incomplete condition: 0
Incomplete system response: 0
Passive voice: 0
Not precise verb: 0
Number Req with smells: 1
```
### Paska Score
PASKA_Score: 95
Progressbar: ![95%](https://progress-bar.dev/95)
### Final Score
Final_score = 73.5
Progressbar: ![73.5%](https://progress-bar.dev/73.5)
output_dataset/Few-shot-learning_0.2temp/Gherkinscenario 19: Inability to impersonate a customer with no account__04_05_2024__14:14:38__100_score.md
+141
View File
@@ -0,0 +1,141 @@
# LLM Experiment Information
* Prefix: Few-shot-learning
* Datetime: 04_05_2024__14:14:38
* UniqueID: d2e46ff8-eae2-403a-949c-ab06abfd8b9c
* Gherkinscenario name: Gherkinscenario 19: Inability to impersonate a customer with no account
* LLM-temperature: 0.2
## Gherkin Input
```
Given: SystemA has customer "harold@thrasher.ie" with first name "Harold"
When: UserA views their details
Then: UserA should be unable to impersonate them
```
## Input prompt, technique: Few-Shot-Learning
### Context
We are translating Gherkin acceptance criteria into Rimay CNL.
For this translation use the following information:
$ACTOR is for example a thing or a person.
$MODAL_VERB is for the following word: must or shall.
$TRIGGER is for example a trigger action, something what a system does.
$WHILE_STRUCTURE is the word 'While' followed by the Gherkin Given condition
$WHEN_STRUCTURE is the word 'When' followed by the Gherkin When condition
$SYSTEM_RESPONSE is a combination of the Actors and the Gherkin Then condition.
$ACTOR, $MODAL_VERB and a $SYSTEM_RESPONSE is mandatory in all Rimay requirements.
Replace the variables with the right text according the input acceptance criteria.
In the following order:
Rimay CNL definition: $WHILE_STRUCTURE $WHEN_STRUCTURE $ACTOR $MODAL_VERB $SYSTEM_RESPONSE
The following example is a valid Rimay CNL:
When SystemB receives an "email alert" from SystemA, SystemB must send an Instruction to SystemC.
With the usage of the following extra information:
Actors: actor SystemA, actor SystemB, actor SystemC, actor UserX, actor UserA, actor UserC
### Input Data
Use the following Gherkin acceptance criteria input:
```
Given: SystemA has customer "harold@thrasher.ie" with first name "Harold"
When: UserA views their details
Then: UserA should be unable to impersonate them
```
### Question:
Can you translate the input above into the specified language Rimay CNL?
### Output indicator
Output only the translated Rimay CNL text and nothing else!
## Rimay Output
```
While SystemA has a customer with email "harold@thrasher.ie" and first name "Harold" When UserA views their details UserA must be unable to impersonate them.
```
### Researcher score
Researcher_score = 15
## DSL-Rimay Verification
```
ERROR:no viable alternative at input 'customer' (line : 3 column : 21)
ERROR:no viable alternative at input 'with' (line : 3 column : 30)
ERROR:no viable alternative at input 'email' (line : 3 column : 35)
ERROR:no viable alternative at input '"harold@thrasher.ie"' (line : 3 column : 41)
ERROR:no viable alternative at input 'first' (line : 3 column : 66)
ERROR:no viable alternative at input 'name' (line : 3 column : 72)
ERROR:missing '.' at 'When' (line : 3 column : 86)
ERROR:no viable alternative at input 'their' (line : 3 column : 103)
ERROR:no viable alternative at input 'details' (line : 3 column : 109)
ERROR:missing '.' at 'UserA' (line : 3 column : 117)
ERROR:no viable alternative at input 'be' (line : 3 column : 128)
ERROR:no viable alternative at input 'unable' (line : 3 column : 131)
ERROR:missing '.' at 'impersonate' (line : 3 column : 141)
ERROR:no viable alternative at input 'them' (line : 3 column : 153)
ERROR:Couldn't resolve reference to Actor 'customer'. (line : 3 column : 21)
ERROR:Couldn't resolve reference to Actor 'email'. (line : 3 column : 35)
ERROR:Couldn't resolve reference to Actor 'first'. (line : 3 column : 66)
ERROR:Couldn't resolve reference to Actor 'views'. (line : 3 column : 97)
ERROR:Couldn't resolve reference to Quantifier 'UserA'. (line : 3 column : 91)
ERROR:Couldn't resolve reference to Actor 'their'. (line : 3 column : 103)
ERROR:Couldn't resolve reference to Actor 'be'. (line : 3 column : 128)
ERROR:Couldn't resolve reference to Actor 'impersonate'. (line : 3 column : 141)
```
### DSL-Rimay Score
DSL_Score: 72
Progressbar: ![72%](https://progress-bar.dev/72)
## Rimay Paska Verification
```
File generated_rimay.csv.csv
Smells found in this document:
Non-atomic requirement: 0
Incomplete requirement: 0
Incorrect order requirement: 0
Coordination ambiguity: 0
Not requirement: ,0
Incomplete condition: 0
Incomplete system response: 0
Passive voice: 0
Not precise verb: 0
Number Req with smells: 0
```
### Paska Score
PASKA_Score: 100
Progressbar: ![100%](https://progress-bar.dev/100)
### Final Score
Final_score = 100
Progressbar: ![100%](https://progress-bar.dev/100)
output_dataset/Few-shot-learning/Scenario 1: Requesting the cash from a creditworthy account_Incorrect___04_02_2024__21:13:40__67.0_score.md → output_dataset/Few-shot-learning_0.2temp/Gherkinscenario 1: Requesting the cash from a creditworthy account_Incorrect___04_02_2024__21:13:40__67.0_score.md
View File
output_dataset/Few-shot-learning/Scenario 1: Requesting the cash from a creditworthy account__04_02_2024__21:12:50__100_score.md → output_dataset/Few-shot-learning_0.2temp/Gherkinscenario 1: Requesting the cash from a creditworthy account__04_02_2024__21:12:50__100_score.md
View File
output_dataset/Few-shot-learning/Scenario 2: Money withdraw from bank account_Incorrect___04_02_2024__21:15:23__61.5_score.md → output_dataset/Few-shot-learning_0.2temp/Gherkinscenario 2: Money withdraw from bank account_Incorrect___04_02_2024__21:15:23__61.5_score.md
View File
output_dataset/Few-shot-learning/Scenario 2: Money withdraw from bank account__04_02_2024__21:14:31__77.5_score.md → output_dataset/Few-shot-learning_0.2temp/Gherkinscenario 2: Money withdraw from bank account__04_02_2024__21:14:31__77.5_score.md
View File
output_dataset/Few-shot-learning/Scenario 3: Requesting the cash from an overdrawn account_Incorrect___04_02_2024__21:17:05__67.5_score.md → output_dataset/Few-shot-learning_0.2temp/Gherkinscenario 3: Requesting the cash from an overdrawn account_Incorrect___04_02_2024__21:17:05__67.5_score.md
View File
output_dataset/Few-shot-learning/Scenario 3: Requesting the cash from an overdrawn account__04_02_2024__21:16:14__96.0_score.md → output_dataset/Few-shot-learning_0.2temp/Gherkinscenario 3: Requesting the cash from an overdrawn account__04_02_2024__21:16:14__96.0_score.md
View File
output_dataset/Few-shot-learning/Scenario 4: User presses button._Incorrect___04_02_2024__21:18:59__69.0_score.md → output_dataset/Few-shot-learning_0.2temp/Gherkinscenario 4: User presses button._Incorrect___04_02_2024__21:18:59__69.0_score.md
View File
output_dataset/Few-shot-learning/Scenario 4: User presses button.__04_02_2024__21:18:01__100_score.md → output_dataset/Few-shot-learning_0.2temp/Gherkinscenario 4: User presses button.__04_02_2024__21:18:01__100_score.md
View File
output_dataset/Few-shot-learning/Scenario 5: Getting verified after clicking the link in the verification message_Incorrect___04_02_2024__21:20:41__57.0_score.md → output_dataset/Few-shot-learning_0.2temp/Gherkinscenario 5: Getting verified after clicking the link in the verification message_Incorrect___04_02_2024__21:20:41__57.0_score.md
View File
output_dataset/Few-shot-learning/Scenario 5: Getting verified after clicking the link in the verification message__04_02_2024__21:19:49__81.5_score.md → output_dataset/Few-shot-learning_0.2temp/Gherkinscenario 5: Getting verified after clicking the link in the verification message__04_02_2024__21:19:49__81.5_score.md
View File
output_dataset/Few-shot-learning/Scenario 6: The selected province stays after validation error_Incorrect___04_02_2024__21:24:43__67.5_score.md → output_dataset/Few-shot-learning_0.2temp/Gherkinscenario 6: The selected province stays after validation error_Incorrect___04_02_2024__21:24:43__67.5_score.md
View File
output_dataset/Few-shot-learning/Scenario 6: The selected province stays after validation error__04_02_2024__21:23:47__81.5_score.md → output_dataset/Few-shot-learning_0.2temp/Gherkinscenario 6: The selected province stays after validation error__04_02_2024__21:23:47__81.5_score.md
View File
output_dataset/Few-shot-learning/Scenario 7: Not being able to add a product to the cart when it is out of stock_Incorrect___04_02_2024__21:27:13__71.5_score.md → output_dataset/Few-shot-learning_0.2temp/Gherkinscenario 7: Not being able to add a product to the cart when it is out of stock_Incorrect___04_02_2024__21:27:13__71.5_score.md
View File
output_dataset/Few-shot-learning/Scenario 7: Not being able to add a product to the cart when it is out of stock__04_02_2024__21:25:57__82.0_score.md → output_dataset/Few-shot-learning_0.2temp/Gherkinscenario 7: Not being able to add a product to the cart when it is out of stock__04_02_2024__21:25:57__82.0_score.md
View File
output_dataset/Few-shot-learning/Scenario 8: Disabling country_Incorrect___04_02_2024__21:29:15__69.0_score.md → output_dataset/Few-shot-learning_0.2temp/Gherkinscenario 8: Disabling country_Incorrect___04_02_2024__21:29:15__69.0_score.md
View File
output_dataset/Few-shot-learning/Scenario 8: Disabling country__04_02_2024__21:28:13__95.5_score.md → output_dataset/Few-shot-learning_0.2temp/Gherkinscenario 8: Disabling country__04_02_2024__21:28:13__95.5_score.md
View File
output_dataset/Few-shot-learning/Scenario 9: Resending the verification email as a logged in user_Incorrect___04_02_2024__21:31:04__70.0_score.md → output_dataset/Few-shot-learning_0.2temp/Gherkinscenario 9: Resending the verification email as a logged in user_Incorrect___04_02_2024__21:31:04__70.0_score.md
View File
output_dataset/Few-shot-learning/Scenario 9: Resending the verification email as a logged in user__04_02_2024__21:30:08__67.5_score.md → output_dataset/Few-shot-learning_0.2temp/Gherkinscenario 9: Resending the verification email as a logged in user__04_02_2024__21:30:08__67.5_score.md
View File
output_dataset/Few-shot-learning_1.3temp/Gherkinscenario 2: Money withdraw from bank account_Incorrect___04_05_2024__13:30:02__63.0_score.md
+148
View File
@@ -0,0 +1,148 @@
# LLM Experiment Information
* Prefix: Few-shot-learning
* Datetime: 04_05_2024__13:30:02
* UniqueID: 6586ae09-fa25-44e1-8464-a5441e413989
* Gherkinscenario name: Gherkinscenario 2: Money withdraw from bank account_Incorrect_
* LLM-temperature: 1.3
## Gherkin Input
```
Given: UserA bank account is in credit
When: UserA attempts to withdraw an amount less than UserA card's limit
Then: the withdrawal should be complete without errors or warnings
```
## Input prompt, technique: Few-Shot-Learning
### Context
We are translating Gherkin acceptance criteria into Rimay CNL.
For this translation use the following information:
$ACTOR is for example a thing or a person.
$MODAL_VERB is for the following word: should or can
$TRIGGER is for example a trigger action, nothing what a system does.
$WHILE_STRUCTUREis the word 'When' followed by the Gherkin When condition
$WHEN_STRUCTURE is the word 'While' followed by the Gherkin Given condition
$SYSTEM_RESPONSE is a combination of the Actors and the Gherkin Then condition.
$ACTOR, $MODAL_VERB and a $SYSTEM_RESPONSE is mandatory in all Rimay requirements.
Replace the variables with the right text according the input acceptance criteria.
In the following order:
Rimay CNL definition: $SYSTEM_RESPONSE $MODAL_VERB $WHILE_STRUCTURE $ACTOR $WHEN_STRUCTURE
With the usage of the following extra information:
Actors: actor SystemA, actor SystemB, actor SystemC, actor UserX, actor UserA, actor UserC
### Input Data
Use the following Gherkin acceptance criteria input:
```
Given: UserA bank account is in credit
When: UserA attempts to withdraw an amount less than UserA card's limit
Then: the withdrawal should be complete without errors or warnings
```
### Question:
Can you translate the input above into the specified language Rimay CNL?
### Output indicator
Output only the translated Rimay CNL text and nothing else!
## Rimay Output
```
The withdrawal should be complete without errors or warnings should While UserA bank account is in credit UserA When UserA attempts to withdraw an amount less than UserA card's limit
```
### Researcher score
Researcher_score = -10
## DSL-Rimay Verification
```
ERROR:no viable alternative at input 'should' (line : 3 column : 16)
ERROR:missing '.' at 'be' (line : 3 column : 23)
ERROR:no viable alternative at input 'complete' (line : 3 column : 26)
ERROR:missing '.' at 'without' (line : 3 column : 35)
ERROR:no viable alternative at input 'errors' (line : 3 column : 43)
ERROR:missing '.' at 'warnings' (line : 3 column : 53)
ERROR:no viable alternative at input 'should' (line : 3 column : 63)
ERROR:missing '.' at 'While' (line : 3 column : 75)
ERROR:no viable alternative at input 'account' (line : 3 column : 92)
ERROR:no viable alternative at input 'is' (line : 3 column : 100)
ERROR:no viable alternative at input 'credit' (line : 3 column : 106)
ERROR:no viable alternative at input 'UserA' (line : 3 column : 117)
ERROR:missing '.' at 'When' (line : 3 column : 126)
ERROR:no viable alternative at input 'to' (line : 3 column : 146)
ERROR:missing ',' at 'withdraw' (line : 3 column : 149)
ERROR:no viable alternative at input 'an' (line : 3 column : 158)
ERROR:missing '.' at 'amount' (line : 3 column : 161)
ERROR:no viable alternative at input 'less' (line : 3 column : 168)
ERROR:no viable alternative at input 'UserA' (line : 3 column : 178)
ERROR:no viable alternative at input 'card' (line : 3 column : 184)
ERROR:mismatched character '<EOF>' expecting ''' (line : 3 column : 188)
ERROR:Couldn't resolve reference to Actor 'withdrawal'. (line : 3 column : 5)
ERROR:Couldn't resolve reference to Actor 'be'. (line : 3 column : 23)
ERROR:Couldn't resolve reference to Actor 'without'. (line : 3 column : 35)
ERROR:Couldn't resolve reference to Actor 'warnings'. (line : 3 column : 53)
ERROR:Couldn't resolve reference to ActorOrClassOrProperty 'bank'. (line : 3 column : 87)
ERROR:Couldn't resolve reference to Quantifier 'UserA'. (line : 3 column : 81)
ERROR:Couldn't resolve reference to Actor 'account'. (line : 3 column : 92)
ERROR:Couldn't resolve reference to Actor 'credit'. (line : 3 column : 106)
ERROR:Couldn't resolve reference to Actor 'attempts'. (line : 3 column : 137)
ERROR:Couldn't resolve reference to Quantifier 'UserA'. (line : 3 column : 131)
ERROR:Couldn't resolve reference to Actor 'withdraw'. (line : 3 column : 149)
ERROR:Couldn't resolve reference to Actor 'amount'. (line : 3 column : 161)
```
### DSL-Rimay Score
DSL_Score: 51
Progressbar: ![51%](https://progress-bar.dev/51)
## Rimay Paska Verification
```
File generated_rimay.csv.csv
Smells found in this document:
Non-atomic requirement: 0
Incomplete requirement: 0
Incorrect order requirement: 1
Coordination ambiguity: 0
Not requirement: ,0
Incomplete condition: 0
Incomplete system response: 0
Passive voice: 0
Not precise verb: 0
Number Req with smells: 1
```
### Paska Score
PASKA_Score: 95
Progressbar: ![95%](https://progress-bar.dev/95)
### Final Score
Final_score = 63.0
Progressbar: ![63.0%](https://progress-bar.dev/63.0)
output_dataset/Few-shot-learning_1.3temp/Gherkinscenario 2: Money withdraw from bank account__04_05_2024__13:29:05__92.5_score.md
+138
View File
@@ -0,0 +1,138 @@
# LLM Experiment Information
* Prefix: Few-shot-learning
* Datetime: 04_05_2024__13:29:05
* UniqueID: 8f138f36-7983-4012-a4be-07be403c98f2
* Gherkinscenario name: Gherkinscenario 2: Money withdraw from bank account
* LLM-temperature: 1.3
## Gherkin Input
```
Given: UserA bank account is in credit
When: UserA attempts to withdraw an amount less than UserA card's limit
Then: the withdrawal should be complete without errors or warnings
```
## Input prompt, technique: Few-Shot-Learning
### Context
We are translating Gherkin acceptance criteria into Rimay CNL.
For this translation use the following information:
$ACTOR is for example a thing or a person.
$MODAL_VERB is for the following word: must or shall.
$TRIGGER is for example a trigger action, something what a system does.
$WHILE_STRUCTURE is the word 'While' followed by the Gherkin Given condition
$WHEN_STRUCTURE is the word 'When' followed by the Gherkin When condition
$SYSTEM_RESPONSE is a combination of the Actors and the Gherkin Then condition.
$ACTOR, $MODAL_VERB and a $SYSTEM_RESPONSE is mandatory in all Rimay requirements.
Replace the variables with the right text according the input acceptance criteria.
In the following order:
Rimay CNL definition: $WHILE_STRUCTURE $WHEN_STRUCTURE $ACTOR $MODAL_VERB $SYSTEM_RESPONSE
The following example is a valid Rimay CNL:
When SystemB receives an "email alert" from SystemA, SystemB must send an Instruction to SystemC.
With the usage of the following extra information:
Actors: actor SystemA, actor SystemB, actor SystemC, actor UserX, actor UserA, actor UserC
### Input Data
Use the following Gherkin acceptance criteria input:
```
Given: UserA bank account is in credit
When: UserA attempts to withdraw an amount less than UserA card's limit
Then: the withdrawal should be complete without errors or warnings
```
### Question:
Can you translate the input above into the specified language Rimay CNL?
### Output indicator
Output only the translated Rimay CNL text and nothing else!
## Rimay Output
```
While UserA bank account is in credit When UserA attempts to withdraw an amount less than UserA card's limit UserA must the withdrawal should be complete without errors or warnings
```
### Researcher score
Researcher_score = 10
## DSL-Rimay Verification
```
ERROR:no viable alternative at input 'account' (line : 3 column : 18)
ERROR:no viable alternative at input 'is' (line : 3 column : 26)
ERROR:no viable alternative at input 'credit' (line : 3 column : 32)
ERROR:no viable alternative at input 'When' (line : 3 column : 41)
ERROR:missing '.' at 'UserA' (line : 3 column : 46)
ERROR:no viable alternative at input 'attempts' (line : 3 column : 52)
ERROR:missing '.' at 'withdraw' (line : 3 column : 64)
ERROR:no viable alternative at input 'an' (line : 3 column : 73)
ERROR:missing '.' at 'amount' (line : 3 column : 76)
ERROR:no viable alternative at input 'less' (line : 3 column : 83)
ERROR:no viable alternative at input 'UserA' (line : 3 column : 93)
ERROR:no viable alternative at input 'card' (line : 3 column : 99)
ERROR:mismatched character '<EOF>' expecting ''' (line : 3 column : 103)
ERROR:Couldn't resolve reference to ActorOrClassOrProperty 'bank'. (line : 3 column : 13)
ERROR:Couldn't resolve reference to Quantifier 'UserA'. (line : 3 column : 7)
ERROR:Couldn't resolve reference to Actor 'account'. (line : 3 column : 18)
ERROR:Couldn't resolve reference to Actor 'credit'. (line : 3 column : 32)
ERROR:Couldn't resolve reference to Actor 'withdraw'. (line : 3 column : 64)
ERROR:Couldn't resolve reference to Actor 'amount'. (line : 3 column : 76)
```
### DSL-Rimay Score
DSL_Score: 65
Progressbar: ![65%](https://progress-bar.dev/65)
## Rimay Paska Verification
```
File generated_rimay.csv.csv
Smells found in this document:
Non-atomic requirement: 0
Incomplete requirement: 0
Incorrect order requirement: 0
Coordination ambiguity: 0
Not requirement: ,0
Incomplete condition: 0
Incomplete system response: 0
Passive voice: 0
Not precise verb: 0
Number Req with smells: 0
```
### Paska Score
PASKA_Score: 100
Progressbar: ![100%](https://progress-bar.dev/100)
### Final Score
Final_score = 92.5
Progressbar: ![92.5%](https://progress-bar.dev/92.5)
prompt_techniques.py
+3 -3
View File
@@ -5,7 +5,7 @@ class LLM_prompt_data():
def __str__(self) -> str:
return self.text
def precondition_data(self): #todo, for gherkin i guess
pass
@@ -94,7 +94,7 @@ $ACTOR, $MODAL_VERB and a $SYSTEM_RESPONSE is mandatory in all Rimay requirement
Replace the variables with the right text according the input acceptance criteria.
In the following order:
Rimay CNL definition: $SYSTEM_RESPONSE $MODAL_VERB $WHILE_STRUCTURE $ACTOR $WHEN_STRUCTURE
Rimay CNL definition: $SYSTEM_RESPONSE $MODAL_VERB $WHILE_STRUCTURE $ACTOR $WHEN_STRUCTURE
With the usage of the following extra information:
Actors: actor SystemA, actor SystemB, actor SystemC, actor UserX, actor UserA, actor UserC
@@ -118,7 +118,7 @@ $ACTOR, $MODAL_VERB and a $SYSTEM_RESPONSE is mandatory in all Rimay requirement
Replace the variables with the right text according the input acceptance criteria.
In the following order:
Rimay CNL definition: $WHILE_STRUCTURE $WHEN_STRUCTURE $ACTOR $MODAL_VERB $SYSTEM_RESPONSE
Rimay CNL definition: $WHILE_STRUCTURE $WHEN_STRUCTURE $ACTOR $MODAL_VERB $SYSTEM_RESPONSE
The following example is a valid Rimay CNL:
standalone_rimay.py
+10 -10
View File
@@ -17,8 +17,8 @@ Classes: class Instruction := description record, class Y
"""
def convert_single_gherkin_to_rimay(scenario_name: str, input_text:str, translation_type: str, incorrect):
current_logger = ResearchLogger(translation_type, scenario_name) #Vervalt bij nieuwe logger.
def convert_single_gherkin_to_rimay(scenario_name: str, input_text:str, translation_type: str, incorrect, temperature):
current_logger = ResearchLogger(translation_type, temperature, scenario_name) #Vervalt bij nieuwe logger.
pre_content = f"""
# Scenario Name {scenario_name}
@@ -32,7 +32,7 @@ def convert_single_gherkin_to_rimay(scenario_name: str, input_text:str, translat
```
"""
current_logger.append_result(pre_content)
response = ask_different_prompts(input_text.strip(), translation_type, current_logger, incorrect)
response = ask_different_prompts(input_text.strip(), translation_type, current_logger, incorrect, temperature)
researcher_score(input_text, response, current_logger)
rimay_check(response, current_logger)
@@ -75,9 +75,9 @@ def rimay_check(rimay_text, current_logger: ResearchLogger):
def start_gherkin_translation(scenario_name: str, acceptance_criteria: str, techniek):
convert_single_gherkin_to_rimay(scenario_name, acceptance_criteria, techniek, False)
convert_single_gherkin_to_rimay(scenario_name + "_Incorrect_", acceptance_criteria, techniek, True)
def start_gherkin_translation(scenario_name: str, acceptance_criteria: str, techniek, llmtemp):
convert_single_gherkin_to_rimay(scenario_name, acceptance_criteria, techniek, False, llmtemp)
convert_single_gherkin_to_rimay(scenario_name + "_Incorrect_", acceptance_criteria, techniek, True, llmtemp)
# convert_single_gherkin_to_rimay(scenario_name, acceptance_criteria, "Chain-of-thought")
# convert_single_gherkin_to_rimay(scenario_name, acceptance_criteria, "Role play")
@@ -103,11 +103,11 @@ def results(path):
score = re.findall(r'\d+', all_data[3])[0]
if "_Incorrect_" in x:
scenarios_TN.append(f"Scenario {scenario_num}")
scenarios_TN.append(f"Generated Rimay {scenario_num}")
data_values_TN.append(int(score))
else:
scenarios_TP.append(f"Scenario {scenario_num}")
scenarios_TP.append(f"Generated Rimay {scenario_num}")
data_values_TP.append(int(score))
@@ -131,8 +131,8 @@ def results(path):
'chart': {'type': 'bar'},
'xAxis': {'categories': scenarios_TP},
'series': [
{'name': path.replace("output_dataset/", "").replace("/", "") + " TP", 'data': data_values_TP},
{'name': path.replace("output_dataset/", "").replace("/", "") + " TN", 'data': data_values_TN},
{'name': path.replace("output_dataset/", "").replace("/", "").replace("_", " ") + " TP", 'data': data_values_TP},
{'name': path.replace("output_dataset/", "").replace("/", "").replace("_", " ") + " TN", 'data': data_values_TN},
# {'name': 'Few-Shot-learning Incorrect (TN)', 'data': [12, 13, 14]},
wrapper_calls.py
+4 -4
View File
@@ -3,7 +3,7 @@ from prompt_techniques import LLM_chain_of_thought, LLM_few_shot_learning, LLM_p
from logger import ResearchLogger
def ask_different_prompts(input_text: str, prompt_tech: str, log:ResearchLogger, incorrect: bool = False) -> str:
def ask_different_prompts(input_text: str, prompt_tech: str, log:ResearchLogger, temperature, incorrect: bool = False) -> str:
prompt_data = LLM_prompt_data(input_text)
llm_communicator = LLM_Communicator()
fsl_prompt = LLM_few_shot_learning(prompt_data, incorrect)
@@ -11,11 +11,11 @@ def ask_different_prompts(input_text: str, prompt_tech: str, log:ResearchLogger,
rp_prompt = LLM_role_play(prompt_data, incorrect)
result = ""
if prompt_tech.strip() == "Few-shot-learning": #['Few-shot learning', 'Chain-of-thought', 'Role play']
result = llm_communicator.ask_llm_to_convert(fsl_prompt)
result = llm_communicator.ask_llm_to_convert(fsl_prompt, temperature)
elif prompt_tech.strip() == "Chain-of-thought":
result = llm_communicator.ask_llm_to_convert(cot_prompt)
result = llm_communicator.ask_llm_to_convert(cot_prompt, temperature)
else: #Role play
result = llm_communicator.ask_llm_to_convert(rp_prompt)
result = llm_communicator.ask_llm_to_convert(rp_prompt, temperature)
llm_communicator.write_log_output(log)