ප්රයිමර් EZ අත්හදා බැලීම

15from labml import experiment
16from labml.configs import option
17from labml_nn.transformers import TransformerConfigs
18from labml_nn.transformers.basic.autoregressive_experiment import Configs
19from labml_nn.transformers.configs import FeedForwardConfigs
20from labml_nn.transformers.primer_ez import SquaredReLU

23@option(FeedForwardConfigs.activation, 'SquaredReLU')
24def _squared_relu():

30    return SquaredReLU()

33@option(TransformerConfigs.encoder_attn, 'MultiDConvHeadAttention')
34def _d_conv_mha(c: TransformerConfigs):

40    from labml_nn.transformers.primer_ez import MultiDConvHeadAttention
41    return MultiDConvHeadAttention(c.n_heads, c.d_model, dropout_prob=c.dropout)

44@option(TransformerConfigs.encoder_attn, 'MultiDSharedConvHeadAttention')
45def _d_shared_conv_mha(c: TransformerConfigs):

53    from labml_nn.transformers.primer_ez.variations import MultiDSharedConvHeadAttention
54    return MultiDSharedConvHeadAttention(c.n_heads, c.d_model, dropout_prob=c.dropout)

57@option(TransformerConfigs.encoder_attn, 'MultiDPHConvHeadAttention')
58def _d_per_head_conv_mha(c: TransformerConfigs):

66    from labml_nn.transformers.primer_ez.variations import MultiDPHConvHeadAttention
67    return MultiDPHConvHeadAttention(c.n_heads, c.d_model, dropout_prob=c.dropout)

70def main():

72    experiment.create(name="primer_ez")

74    conf = Configs()

76    experiment.configs(conf, {

78        'tokenizer': 'character',

80        'prompt_separator': '',

82        'prompt': 'It is ',

84        'text': 'tiny_shakespeare',

87        'seq_len': 256,

89        'epochs': 128,

91        'batch_size': 32,

94        'inner_iterations': 10,

97        'd_model': 512,
98        'transformer.ffn.d_ff': 2048,

101        'optimizer.optimizer': 'Adam',
102        'optimizer.learning_rate': 2.5e-4,

107        'transformer.ffn.activation': 'SquaredReLU',

112        'transformer.encoder_attn': 'MultiDConvHeadAttention',
113    })

116    experiment.add_pytorch_models({'model': conf.model})

119    with experiment.start():

121        conf.run()

125if __name__ == '__main__':
126    main()