This commit is contained in:
游雁 2024-03-26 16:47:10 +08:00
parent 507f821d7a
commit 46ec82c054
5 changed files with 5 additions and 5 deletions

View File

@ -96,7 +96,7 @@ dataset_conf:
index_ds: IndexDSJsonl
batch_sampler: BatchSampler
batch_type: example # example or length
batch_size: 32 # if batch_type is example, batch_size is the numbers of samples; if length, batch_size is source_token_len+target_token_len;
batch_size: 16 # if batch_type is example, batch_size is the numbers of samples; if length, batch_size is source_token_len+target_token_len;
max_token_length: 2048 # filter samples if source_token_len+target_token_len > max_token_length,
buffer_size: 1024
shuffle: True

View File

@ -89,7 +89,7 @@ dataset_conf:
index_ds: IndexDSJsonl
batch_sampler: BatchSampler
batch_type: example # example or length
batch_size: 32 # if batch_type is example, batch_size is the numbers of samples; if length, batch_size is source_token_len+target_token_len;
batch_size: 16 # if batch_type is example, batch_size is the numbers of samples; if length, batch_size is source_token_len+target_token_len;
max_token_length: 2048 # filter samples if source_token_len+target_token_len > max_token_length,
buffer_size: 1024
shuffle: True

View File

@ -96,7 +96,7 @@ dataset_conf:
index_ds: IndexDSJsonl
batch_sampler: BatchSampler
batch_type: example # example or length
batch_size: 32 # if batch_type is example, batch_size is the numbers of samples; if length, batch_size is source_token_len+target_token_len;
batch_size: 16 # if batch_type is example, batch_size is the numbers of samples; if length, batch_size is source_token_len+target_token_len;
max_token_length: 2048 # filter samples if source_token_len+target_token_len > max_token_length,
buffer_size: 1024
shuffle: True

View File

@ -95,7 +95,7 @@ dataset_conf:
index_ds: IndexDSJsonl
batch_sampler: BatchSampler
batch_type: example # example or length
batch_size: 32 # if batch_type is example, batch_size is the numbers of samples; if length, batch_size is source_token_len+target_token_len;
batch_size: 16 # if batch_type is example, batch_size is the numbers of samples; if length, batch_size is source_token_len+target_token_len;
max_token_length: 2048 # filter samples if source_token_len+target_token_len > max_token_length,
buffer_size: 1024
shuffle: True

View File

@ -83,7 +83,7 @@ dataset_conf:
index_ds: IndexDSJsonl
batch_sampler: BatchSampler
batch_type: example # example or length
batch_size: 32 # if batch_type is example, batch_size is the numbers of samples; if length, batch_size is source_token_len+target_token_len;
batch_size: 16 # if batch_type is example, batch_size is the numbers of samples; if length, batch_size is source_token_len+target_token_len;
max_token_length: 2048 # filter samples if source_token_len+target_token_len > max_token_length,
buffer_size: 1024
shuffle: True