සාපේක්ෂදුර (ROPER) අත්හදා බැලීම සහිත රොටරි ස්ථානීය කාවැද්දීම්

11from labml import experiment
12from labml.configs import calculate
13from labml_nn.experiments.arithmetic_dataset import ArithmeticAutoregression
14from labml_nn.transformers import TransformerConfigs
15from labml_nn.transformers.rope.experiment import Configs as RoPEConfigs

18class Configs(RoPEConfigs, ArithmeticAutoregression):

26    pass

29def _rotary_value_pe_mha(c: TransformerConfigs):

33    from labml_nn.transformers.rope.value_pe import RotaryValuePEMultiHeadAttention
34    return RotaryValuePEMultiHeadAttention(c.n_heads, c.d_model, 1., 1.)

38calculate(TransformerConfigs.encoder_attn, 'rotary_value', _rotary_value_pe_mha)
39calculate(TransformerConfigs.decoder_attn, 'rotary_value', _rotary_value_pe_mha)
40calculate(TransformerConfigs.decoder_mem_attn, 'rotary_value', _rotary_value_pe_mha)

43def main():

45    experiment.create(name="roper_addition", comment="rotary value 7", writers={'screen', 'labml'})

47    conf = Configs()

49    experiment.configs(conf, {
50        'max_digits': 7,

53        'transformer.src_embed': 'no_pos',
54        'transformer.tgt_embed': 'no_pos',

57        'transformer.encoder_attn': 'rotary_value',

62        'model': 'rotary_pe_transformer',

65        'seq_len': 512,

67        'epochs': 20,

69        'batch_size': 16,

72        'd_model': 128,
73        'transformer.ffn.d_ff': 512,
74        'transformer.n_heads': 4,
75        'transformer.dropout': 0.0,

78        'optimizer.optimizer': 'Adam',
79        'optimizer.learning_rate': 2.5e-4,
80    })

83    experiment.add_pytorch_models({'model': conf.model})

86    with experiment.start():

88        conf.run()

92if __name__ == '__main__':
93    main()