|
--- |
|
metrics: |
|
- bertscore |
|
- accuracy |
|
model-index: |
|
- name: StarCoder |
|
results: |
|
- task: |
|
type: text-generation |
|
dataset: |
|
type: openai_humaneval |
|
name: HumanEval (Prompted) |
|
metrics: |
|
- name: pass@1 |
|
type: pass@1 |
|
value: 0.408 |
|
verified: false |
|
- task: |
|
type: text-generation |
|
dataset: |
|
type: openai_humaneval |
|
name: HumanEval |
|
metrics: |
|
- name: pass@1 |
|
type: pass@1 |
|
value: 0.336 |
|
verified: false |
|
- task: |
|
type: text-generation |
|
dataset: |
|
type: mbpp |
|
name: MBPP |
|
metrics: |
|
- name: pass@1 |
|
type: pass@1 |
|
value: 0.527 |
|
verified: false |
|
- task: |
|
type: text-generation |
|
dataset: |
|
type: ds1000 |
|
name: DS-1000 (Overall Completion) |
|
metrics: |
|
- name: pass@1 |
|
type: pass@1 |
|
value: 0.26 |
|
verified: false |
|
- task: |
|
type: text-generation |
|
dataset: |
|
type: nuprl/MultiPL-E |
|
name: MultiPL-HumanEval (C++) |
|
metrics: |
|
- name: pass@1 |
|
type: pass@1 |
|
value: 0.3155 |
|
verified: false |
|
- task: |
|
type: text-generation |
|
dataset: |
|
type: nuprl/MultiPL-E |
|
name: MultiPL-HumanEval (C#) |
|
metrics: |
|
- name: pass@1 |
|
type: pass@1 |
|
value: 0.2101 |
|
verified: false |
|
- task: |
|
type: text-generation |
|
dataset: |
|
type: nuprl/MultiPL-E |
|
name: MultiPL-HumanEval (D) |
|
metrics: |
|
- name: pass@1 |
|
type: pass@1 |
|
value: 0.1357 |
|
verified: false |
|
- task: |
|
type: text-generation |
|
dataset: |
|
type: nuprl/MultiPL-E |
|
name: MultiPL-HumanEval (Go) |
|
metrics: |
|
- name: pass@1 |
|
type: pass@1 |
|
value: 0.1761 |
|
verified: false |
|
- task: |
|
type: text-generation |
|
dataset: |
|
type: nuprl/MultiPL-E |
|
name: MultiPL-HumanEval (Java) |
|
metrics: |
|
- name: pass@1 |
|
type: pass@1 |
|
value: 0.3022 |
|
verified: false |
|
- task: |
|
type: text-generation |
|
dataset: |
|
type: nuprl/MultiPL-E |
|
name: MultiPL-HumanEval (Julia) |
|
metrics: |
|
- name: pass@1 |
|
type: pass@1 |
|
value: 0.2302 |
|
verified: false |
|
- task: |
|
type: text-generation |
|
dataset: |
|
type: nuprl/MultiPL-E |
|
name: MultiPL-HumanEval (JavaScript) |
|
metrics: |
|
- name: pass@1 |
|
type: pass@1 |
|
value: 0.3079 |
|
verified: false |
|
- task: |
|
type: text-generation |
|
dataset: |
|
type: nuprl/MultiPL-E |
|
name: MultiPL-HumanEval (Lua) |
|
metrics: |
|
- name: pass@1 |
|
type: pass@1 |
|
value: 0.2389 |
|
verified: false |
|
- task: |
|
type: text-generation |
|
dataset: |
|
type: nuprl/MultiPL-E |
|
name: MultiPL-HumanEval (PHP) |
|
metrics: |
|
- name: pass@1 |
|
type: pass@1 |
|
value: 0.2608 |
|
verified: false |
|
- task: |
|
type: text-generation |
|
dataset: |
|
type: nuprl/MultiPL-E |
|
name: MultiPL-HumanEval (Perl) |
|
metrics: |
|
- name: pass@1 |
|
type: pass@1 |
|
value: 0.1734 |
|
verified: false |
|
- task: |
|
type: text-generation |
|
dataset: |
|
type: nuprl/MultiPL-E |
|
name: MultiPL-HumanEval (Python) |
|
metrics: |
|
- name: pass@1 |
|
type: pass@1 |
|
value: 0.3357 |
|
verified: false |
|
- task: |
|
type: text-generation |
|
dataset: |
|
type: nuprl/MultiPL-E |
|
name: MultiPL-HumanEval (R) |
|
metrics: |
|
- name: pass@1 |
|
type: pass@1 |
|
value: 0.155 |
|
verified: false |
|
- task: |
|
type: text-generation |
|
dataset: |
|
type: nuprl/MultiPL-E |
|
name: MultiPL-HumanEval (Ruby) |
|
metrics: |
|
- name: pass@1 |
|
type: pass@1 |
|
value: 0.0124 |
|
verified: false |
|
- task: |
|
type: text-generation |
|
dataset: |
|
type: nuprl/MultiPL-E |
|
name: MultiPL-HumanEval (Racket) |
|
metrics: |
|
- name: pass@1 |
|
type: pass@1 |
|
value: 0.0007 |
|
verified: false |
|
- task: |
|
type: text-generation |
|
dataset: |
|
type: nuprl/MultiPL-E |
|
name: MultiPL-HumanEval (Rust) |
|
metrics: |
|
- name: pass@1 |
|
type: pass@1 |
|
value: 0.2184 |
|
verified: false |
|
- task: |
|
type: text-generation |
|
dataset: |
|
type: nuprl/MultiPL-E |
|
name: MultiPL-HumanEval (Scala) |
|
metrics: |
|
- name: pass@1 |
|
type: pass@1 |
|
value: 0.2761 |
|
verified: false |
|
- task: |
|
type: text-generation |
|
dataset: |
|
type: nuprl/MultiPL-E |
|
name: MultiPL-HumanEval (Bash) |
|
metrics: |
|
- name: pass@1 |
|
type: pass@1 |
|
value: 0.1046 |
|
verified: false |
|
- task: |
|
type: text-generation |
|
dataset: |
|
type: nuprl/MultiPL-E |
|
name: MultiPL-HumanEval (Swift) |
|
metrics: |
|
- name: pass@1 |
|
type: pass@1 |
|
value: 0.2274 |
|
verified: false |
|
- task: |
|
type: text-generation |
|
dataset: |
|
type: nuprl/MultiPL-E |
|
name: MultiPL-HumanEval (TypeScript) |
|
metrics: |
|
- name: pass@1 |
|
type: pass@1 |
|
value: 0.3229 |
|
verified: false |
|
--- |
