රොටරිස්ථානීය කාවැද්දීම් (කඹය) අත්හදා බැලීම

14from labml import experiment
15from labml.configs import calculate
16from labml_nn.transformers import TransformerConfigs
17from labml_nn.transformers.rope.experiment import Configs as RoPEConfigs

22class Configs(RoPEConfigs):  # , ArithmeticAutoregression):
23    pass

26def _rotary_value_pe_mha(c: TransformerConfigs):
27    from labml_nn.transformers.rope.value_pe import RotaryValuePEMultiHeadAttention
28    return RotaryValuePEMultiHeadAttention(c.n_heads, c.d_model, 1., 1.)

32calculate(TransformerConfigs.encoder_attn, 'rotary_value', _rotary_value_pe_mha)
33calculate(TransformerConfigs.decoder_attn, 'rotary_value', _rotary_value_pe_mha)
34calculate(TransformerConfigs.decoder_mem_attn, 'rotary_value', _rotary_value_pe_mha)

37def main():

39    experiment.create(name="rotary_shakespeare", comment="rotary value", writers={'screen', 'labml'})

41    conf = Configs()

43    experiment.configs(conf, {

45        'transformer.src_embed': 'no_pos',
46        'transformer.tgt_embed': 'no_pos',

49        'transformer.encoder_attn': 'rotary_value',

53        'model': 'rotary_pe_transformer',

56        'tokenizer': 'character',

58        'prompt_separator': '',

60        'prompt': 'It is ',

62        'text': 'tiny_shakespeare',

65        'seq_len': 512,

67        'epochs': 24,

69        'batch_size': 16,

72        'inner_iterations': 4,

75        'd_model': 128,
76        'transformer.ffn.d_ff': 512,
77        'transformer.n_heads': 4,
78        'transformer.dropout': 0.0,

81        'optimizer.optimizer': 'Adam',
82        'optimizer.learning_rate': 2.5e-4,
83
84        'dataloader_shuffle_with_replacement': True
85    })

88    experiment.add_pytorch_models({'model': conf.model})

91    with experiment.start():

93        conf.run()

97if __name__ == '__main__':
98    main()