```python
instruction = '[IMAGE:img] what does the image describe? -> [TEXT:cap] '
data = {'img': "https://modelscope.oss-cn-beijing.aliyuncs.com/demo/images/ic.jpeg"}
output = pipe(data, instruction=instruction)
print(output.text) # "a man and woman sitting in front of a laptop computer"
```
### Visual Grounding
```python
instruction = '[IMAGE:img] which region does the text " [TEXT:cap] " describe? -> [BOX:patch_boxes,add_bos,add_eos]'
data = {'img': "https://modelscope.oss-cn-beijing.aliyuncs.com/demo/images/vg.jpg", "cap": "hand"}
output = pipe(data, instruction=instruction)
output.save_box("output.jpg")
```
### Text Summarization
```python
instruction = ' what is the summary of article " [TEXT:src] "? -> [TEXT:tgt] '
data = {'src': "poland 's main opposition party tuesday endorsed president lech walesa in an upcoming "
"presidential run-off election after a reformed communist won the first round of voting ."}
output = pipe(data, instruction=instruction)
print(output.text) # "polish opposition endorses walesa in presidential run-off"
```
### Table-to-Text Generation
```python
instruction = ' structured knowledge: " [STRUCT:database,uncased] " . how to describe the tripleset ? -> [TEXT:tgt] '
data = {
'database': [['Atlanta', 'OFFICIAL_POPULATION', '5,457,831'],
['[TABLECONTEXT]', 'METROPOLITAN_AREA', 'Atlanta'],
['5,457,831', 'YEAR', '2012'],
['[TABLECONTEXT]', '[TITLE]', 'List of metropolitan areas by population'],
['Atlanta', 'COUNTRY', 'United States'],
]
}
output = pipe(data, instruction=instruction, beam_size=1)
print(output.text) # "atlanta is the metropolitan area in the united states in 2012."
```
### Text-to-SQL Generation
```python
instruction = ' " [TEXT:src] " ; structured knowledge: " [STRUCT:database,max_length=876] " . generating sql code. -> [TEXT:tgt] '
database = [
['concert_singer'],
['stadium', 'stadium_id , location , name , capacity , highest , lowest , average'],
['singer', 'singer_id , name , country , song_name , song_release_year , age , is_male'],
['concert', 'concert_id , concert_name , theme , stadium_id , year'],
['singer_in_concert', 'concert_id , singer_id']
]
data = [
{'src': 'What are the names, countries, and ages for every singer in descending order of age?', 'database': database},
{'src': 'What are all distinct countries where singers above age 20 are from?', 'database': database},
{'src': 'What are the locations and names of all stations with capacity between 5000 and 10000?', 'database': database}
]
output = pipe(data, instruction=instruction)
print('\n'.join([o.text for o in output]))
# "select name, country, age from singer order by age desc"
# "select distinct country from singer where age > 20"
# "select location, name from stadium where capacity between 5000 and 10000"
```
### Video Captioning
```python
instruction = '[VIDEO:video] what does the video describe? -> [TEXT:cap] '
data = {'video': 'https://modelscope.oss-cn-beijing.aliyuncs.com/demo/videos/video7021.mp4'}
output = pipe(data, instruction=instruction)
print(output.text) # "a baseball player is hitting a ball"
```
### Speech-to-Text Generation
```python
instruction = '[AUDIO:wav] what is the text corresponding to the voice? -> [TEXT:text,preprocess=text_phone,add_bos,add_eos]'
data = {'wav': 'https://modelscope.oss-cn-beijing.aliyuncs.com/demo/audios/1272-128104-0001.flac'}
output = pipe(data, instruction=instruction)
print(output.text) # "nor is mister klohs manner less interesting than his manner"
```
### Text-to-Image Generation
```python
instruction = 'what is the complete image? caption: [TEXT:text]"? -> [IMAGE,preprocess=image_vqgan,adaptor=image_vqgan]'
data = {'text': "a city with tall buildings and a large green park."}
output = pipe(data, instruction=instruction)
output[0].save_image('0.png')
```
## 模型局限性以及可能的偏差
训练数据集自身有局限，有可能产生一些偏差，请用户自行评测后决定如何使用。
## 相关论文以及引用
如果你觉得OFASys好用，喜欢我们的工作，欢迎引用：
```
@article{bai2022ofasys,
author = {
Jinze Bai and
Rui Men and
Hao Yang and
Xuancheng Ren and
Kai Dang and
Yichang Zhang and
Xiaohuan Zhou and
Peng Wang and
Sinan Tan and
An Yang and
Zeyu Cui and
Yu Han and
Shuai Bai and
Wenbin Ge and
Jianxin Ma and
Junyang Lin and
Jingren Zhou and
Chang Zhou},
title = {OFASys: A Multi-Modal Multi-Task Learning System for Building Generalist Models},
journal = {CoRR},
volume = {abs/2212.04408},
year = {2022}
}
```
```python
instruction = '[IMAGE:img] 
```python
instruction = '[IMAGE:img] 
### Text Summarization
```python
instruction = '
## 模型局限性以及可能的偏差
训练数据集自身有局限，有可能产生一些偏差，请用户自行评测后决定如何使用。
## 相关论文以及引用
如果你觉得OFASys好用，喜欢我们的工作，欢迎引用：
```
@article{bai2022ofasys,
author = {
Jinze Bai and
Rui Men and
Hao Yang and
Xuancheng Ren and
Kai Dang and
Yichang Zhang and
Xiaohuan Zhou and
Peng Wang and
Sinan Tan and
An Yang and
Zeyu Cui and
Yu Han and
Shuai Bai and
Wenbin Ge and
Jianxin Ma and
Junyang Lin and
Jingren Zhou and
Chang Zhou},
title = {OFASys: A Multi-Modal Multi-Task Learning System for Building Generalist Models},
journal = {CoRR},
volume = {abs/2212.04408},
year = {2022}
}
```